diff --git a/.github/workflows/joule-moq-room.yml b/.github/workflows/joule-moq-room.yml index a91414949..ebc1574ba 100644 --- a/.github/workflows/joule-moq-room.yml +++ b/.github/workflows/joule-moq-room.yml @@ -37,6 +37,17 @@ name: joule-moq-room room service # named alike (room.grant), a development room (`wai-room dev`) started on loopback, grants # for two test principals, each joining and publishing, the session verified as live-rooms §9 # verifies a room, a changed record refused, and a grant from a service off the machine refused. +# tests/ports.rs holds the local stack's ports in one place: room.example.toml's relay_url is +# relay.example.toml's WebSocket endpoint and path, gateway.example.toml, read by the gateway's +# own reader (joule-moq-ingest, a dev-dependency), has the relay's QUIC endpoint and each listener +# it names a row (off until set when commented out, and read at the row's port once set), the relay's +# example is `wai-relay --dev`, wai dev's origin is the one the remote desktop template's host +# reaches, and the defaults of wai-audit, wai-ledger, wai-fleet, wai-quantum-service, the worlds' +# relay, the stitched presentation verifier and `wai http origin` are their sources', every one +# of them a row of wai/quickstart/README.md's port table and no port twice; the room service +# started from its example opens its clock's session on the relay started from its; and the +# gateway started from its example (ports moved to free ones, its files made) opens HTTP and +# RTMP and none of the listeners its example leaves off. # # It builds on wai-rs's room, event, composition, co-watch and remote desktop records, on the # relay, on joule-moq-mls and on the remote desktop host agent, so a change to any of them runs @@ -58,6 +69,17 @@ on: - 'joule-code/joule-code-rs/vendor/**' - 'wai/wai-rs/**' - 'wai/wai-ops/**' + # Read and started by tests/ports.rs: the ingest gateway and what it builds on. + - 'joule-code/joule-code-rs/crates/joule-moq-ingest/**' + - 'joule-code/joule-code-rs/crates/joule-contrib/**' + # Read by tests/ports.rs. + - 'wai/quickstart/README.md' + - 'wai/examples/remote-desktop/host.sh' + - 'wai/wai-audit/src/main.rs' + - 'wai/wai-ledger/src/main.rs' + - 'wai/wai-fleet/src/main.rs' + - 'wai/wai-quantum-service/src/main.rs' + - 'wai/wai-relay/src/main.rs' - '.github/workflows/joule-moq-room.yml' jobs: diff --git a/.github/workflows/wai-catalogue.yml b/.github/workflows/wai-catalogue.yml index c2859bc87..f7c5a56b8 100644 --- a/.github/workflows/wai-catalogue.yml +++ b/.github/workflows/wai-catalogue.yml @@ -25,6 +25,8 @@ on: - 'wai/wai-rs/**' - 'wai/catalogue/**' - 'wai/examples/**' + # The project file's reader (dx/project.rs, in the binary's tests) holds wai.json's schema to it. + - 'wai/schema/**' - 'wai/http-conformance/**' - 'wai/score-conformance/**' - 'wai/live-conformance/fleet/**' @@ -48,6 +50,8 @@ on: - 'wai/wai-rs/**' - 'wai/catalogue/**' - 'wai/examples/**' + # The project file's reader (dx/project.rs, in the binary's tests) holds wai.json's schema to it. + - 'wai/schema/**' - 'wai/http-conformance/**' - 'wai/score-conformance/**' - 'wai/live-conformance/fleet/**' diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index 5d9fb631b..650393dfb 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -38,6 +38,8 @@ on: # `wai create` holds the examples' files (include_bytes!), and tests/cli_dx.rs holds them # to wai/examples/ byte for byte. - 'wai/examples/**' + # The project file's reader (dx/project.rs, in the binary's tests) holds wai.json's schema to it. + - 'wai/schema/**' # wai-rs path-depends on wai-quantum, and the quantum corpora below are # verified through it: a change there changes what this workflow checks. - 'wai/wai-quantum/**' diff --git a/.github/workflows/wai-examples.yml b/.github/workflows/wai-examples.yml index 3438289f9..cf28ff14e 100644 --- a/.github/workflows/wai-examples.yml +++ b/.github/workflows/wai-examples.yml @@ -7,10 +7,17 @@ name: WAI examples # - portable (linux): remote-desktop (a drawn screen shared into a room, the session accepted # by the independent verifier), package-and-deliver (packed and verified), verify-receipts # (the Rust verifier on this checkout's wai crate and the Python one, each refusing a changed -# byte), encoder-ingest (SRT into a room, every group verified against its receipt). -# - browser (macos): the room client built and packaged as the release packages it -# (wai/quickstart/build-web.sh, build.sh); the quickstart's readiness check, `wai dev -# --check`, from the unpacked archive in an empty directory; then video-room, watch-together, +# byte), encoder-ingest (SRT into a room, every group verified against its receipt); and the +# package managers' entries the release writes (wai/quickstart/test/formulae_test.py): every +# binary of the archive but wai-quantum, which they depend on as its own package, and that +# one built as its own release builds it. +# - browser (macos): the room client and WAI Studio built and packaged as the release packages +# them (wai/quickstart/build-web.sh, build-studio.sh, build.sh, its bin/ with wai-ingest and +# wai-quantum), the checkout left as it was; the quickstart's readiness check, `wai dev --check`, +# from the unpacked archive in an empty directory, Studio at /_wai/ among the pieces it checks; +# the quickstart on a clean machine (wai/quickstart/test/clean-machine.sh: a fresh HOME and +# directory, the system's PATH, every binary and Studio's files in the archive, each binary +# run, then hello, verify, create and dev --check); then video-room, watch-together, # messaging, remote-desktop and package-and-deliver in the host's browser; and apple-app # (the native sink's macOS slice, the SwiftUI app built, a recorded session replayed). # - android (linux-arm64): android-app, the native sink's Kotlin API on the host JVM. @@ -30,6 +37,11 @@ on: - 'wai/wai-pack/**' - 'wai/wai-live-web/**' - 'wai/wai-quantum/**' + - 'wai/wai-quantum-cli/**' + - 'wai/wai-quantum-qpu/**' + - 'wai/wai-studio/**' + - 'wai/quantum-studio/**' + - 'wai/wai-web/**' - 'wai/wai-ops/**' - 'wai/wai-sink-swift/**' - 'wai/wai-sink-android/**' @@ -40,6 +52,9 @@ on: - 'wai/live-conformance/telemetry_aggregate_verify.py' - 'wai/pack-conformance/packs/video-base-01/**' - 'joule-code/joule-code-rs/**' + # Read by wai/quickstart/test/formulae_test.py. + - '.github/workflows/wai-release.yml' + - '.github/workflows/wai-quantum-cli-release.yml' - '.github/workflows/wai-examples.yml' schedule: - cron: '17 5 * * *' @@ -103,6 +118,9 @@ jobs: env: WAI_WEB: ${{ github.workspace }}/joule-code/joule-code-rs/crates/joule-moq-web/web run: bash wai/quickstart/test/run-examples.sh portable + - name: The Homebrew formula and the Scoop manifest the release writes + shell: bash + run: python3 -I wai/quickstart/test/formulae_test.py browser: name: WAI examples · the archive, the readiness check, the browser and Apple · macos @@ -112,20 +130,28 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v5 - - name: Put cargo on PATH + # launchd's PATH omits cargo and node on the macOS host runner. + - name: Put cargo and node on PATH shell: bash - run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + run: | + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + echo "/opt/homebrew/bin" >> "$GITHUB_PATH" - name: wasm32 target and wasm-bindgen CLI 0.2.127 shell: bash run: | rustup target add wasm32-unknown-unknown command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q ' 0\.2\.127$' || cargo install --locked wasm-bindgen-cli --version 0.2.127 - - name: The room client and the archive, as the release builds them + - name: The room client, WAI Studio and the archive, as the release builds them shell: bash run: | set -euo pipefail nice -n 10 bash wai/quickstart/build-web.sh "$RUNNER_TEMP/web" - nice -n 10 bash wai/quickstart/build.sh "$RUNNER_TEMP/dist" --web "$RUNNER_TEMP/web" + nice -n 10 bash wai/quickstart/build-studio.sh "$RUNNER_TEMP/studio" + nice -n 10 bash wai/quickstart/build.sh "$RUNNER_TEMP/dist" --web "$RUNNER_TEMP/web" --studio "$RUNNER_TEMP/studio" + # The three builds leave the checkout as they found it: everything they write is under + # an ignored directory or outside the tree. + left=$(git status --porcelain --untracked-files=all) + [ -z "$left" ] || { echo "$left"; echo "::error::the builds left files in the checkout"; exit 1; } mkdir -p "$RUNNER_TEMP/unpacked" tar -C "$RUNNER_TEMP/unpacked" -xzf "$RUNNER_TEMP"/dist/wai-*.tar.gz echo "WAI_DIST=$(ls -d "$RUNNER_TEMP"/unpacked/wai-*)" >> "$GITHUB_ENV" @@ -138,7 +164,14 @@ jobs: d="$RUNNER_TEMP/clean"; rm -rf "$d"; mkdir -p "$d"; cd "$d" "$WAI_DIST/bin/wai" hello h "$WAI_DIST/bin/wai" verify h - mkdir p && cd p && "$WAI_DIST/bin/wai" dev --check --port 0 + mkdir p && cd p && "$WAI_DIST/bin/wai" dev --check --port 0 | tee "$RUNNER_TEMP/dev-check.out" + grep -q "ok WAI Studio at /_wai/" "$RUNNER_TEMP/dev-check.out" || { echo "::error::the archive's wai dev does not serve WAI Studio"; exit 1; } + # The quickstart as a new user runs it, from the archive and its .sha256 alone, with no + # browser step: it fails when the archive lacks any binary, the room client, or Studio's + # page and core, or when a step fails. + - name: The quickstart on a clean machine, from the archive + shell: bash + run: bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP"/dist/wai-*.tar.gz - name: The examples in the browser shell: bash env: diff --git a/.github/workflows/wai-release.yml b/.github/workflows/wai-release.yml index e34ef27e8..85adabe29 100644 --- a/.github/workflows/wai-release.yml +++ b/.github/workflows/wai-release.yml @@ -3,11 +3,15 @@ name: wai command line release # On a tag wai-v (the version wai/wai-rs/Cargo.toml states), builds the quickstart # archive (wai/quickstart/build.sh) on each runner and attaches it to the forge release of that # tag, each archive with its SHA-256. An archive holds bin/ (wai with the feature quickstart, -# wai-relay, wai-room, wai-remote-host, wai-pack) and share/wai/web/ (the room -# client, built once, on the macOS host, which has the WebAssembly toolchain): +# wai-relay, wai-room, wai-ingest, wai-remote-host, wai-pack, wai-quantum), share/wai/web/ (the +# room client) and share/wai/studio/ (WAI Studio, which `wai dev` serves at /_wai/), both built +# once, on the macOS host, which has the WebAssembly toolchain and Node: # macOS aarch64-apple-darwin, and x86_64-apple-darwin on the Intel Mac (macos-x86_64); # Linux x86_64- and aarch64-unknown-linux-musl, static, so any distribution runs it; # Windows x86_64-pc-windows-msvc, as a zip. +# The first job checks the tag against the crate's version and the installer's, and that each +# project file schema (wai/schema/) is served at its $id as committed: every project file the +# release writes names its schema there, so a tag does not build until the site serves it. # Each archive's own `wai` runs before it is attached (create --list, hello, verify, and on # the hosts that run it, dev --check). A last job writes the Homebrew formula and the Scoop # manifest for the archives attached and attaches them too; they go into the tap and the bucket @@ -50,20 +54,42 @@ jobs: crate="$(sed -n 's/^version = "\(.*\)"$/\1/p' wai/wai-rs/Cargo.toml | head -1)" [ "$v" = "$crate" ] || { echo "::error::the tag says $v, wai/wai-rs/Cargo.toml says $crate"; exit 1; } grep -q "WAI_VERSION:-$v}" wai/quickstart/install.sh || { echo "::error::wai/quickstart/install.sh does not install $v"; exit 1; } + # Every project file this release writes names its schema by URL ($schema), which + # editors and agents fetch: each committed schema is served at its $id, as committed, + # before a release writes a file that names it (wai/schema/README.md). + for f in wai/schema/wai-json-*.schema.json; do + python3 - "$f" <<'PY' || exit 1 + import json, sys, urllib.request + f = sys.argv[1] + want = json.load(open(f)) + url = want["$id"] + try: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "wai-release"}), timeout=30) as r: + got = json.loads(r.read()) + except Exception as e: + sys.exit(f"::error::{url} does not serve {f} ({e}): deploy the site with the schema before the release") + if got != want: + sys.exit(f"::error::{url} serves a schema other than {f}: deploy the site with the schema before the release") + print(f"{url} serves {f}") + PY + done echo "version=$v" >> "$GITHUB_OUTPUT" - # The room client, once: the same files on every platform. + # The room client and WAI Studio, once: the same files on every platform. web: - name: the room client + name: the room client and WAI Studio needs: version runs-on: macos timeout-minutes: 60 steps: - uses: actions/checkout@v5 - - name: Put cargo on PATH + # launchd's PATH omits cargo and node on the macOS host runner. + - name: Put cargo and node on PATH shell: bash - run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - name: Build and attach it + run: | + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + echo "/opt/homebrew/bin" >> "$GITHUB_PATH" + - name: Build and attach them shell: bash env: FORGE_TOKEN: ${{ github.token }} @@ -73,8 +99,9 @@ jobs: command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q ' 0\.2\.127$' || cargo install --locked wasm-bindgen-cli --version 0.2.127 v="${{ needs.version.outputs.version }}" bash wai/quickstart/build-web.sh "$RUNNER_TEMP/web" + bash wai/quickstart/build-studio.sh "$RUNNER_TEMP/studio" mkdir -p "$RUNNER_TEMP/dist" - tar -C "$RUNNER_TEMP" -czf "$RUNNER_TEMP/dist/wai-web-$v.tar.gz" web + tar -C "$RUNNER_TEMP" -czf "$RUNNER_TEMP/dist/wai-web-$v.tar.gz" web studio ( cd "$RUNNER_TEMP/dist" && shasum -a 256 "wai-web-$v.tar.gz" > "wai-web-$v.tar.gz.sha256" ) bash wai/quickstart/publish.sh "$GITHUB_REF_NAME" "wai $v" "$RUNNER_TEMP"/dist/* @@ -104,7 +131,7 @@ jobs: rustup target add "${{ matrix.triple }}" curl -fsS -o "$RUNNER_TEMP/web.tar.gz" "${FORGE_API%/api/v1}/$FORGE_REPO/releases/download/$GITHUB_REF_NAME/wai-web-$v.tar.gz" tar -C "$RUNNER_TEMP" -xzf "$RUNNER_TEMP/web.tar.gz" - nice -n 10 bash wai/quickstart/build.sh "$RUNNER_TEMP/dist" --web "$RUNNER_TEMP/web" --target "${{ matrix.triple }}" --version "$v" + nice -n 10 bash wai/quickstart/build.sh "$RUNNER_TEMP/dist" --web "$RUNNER_TEMP/web" --studio "$RUNNER_TEMP/studio" --target "${{ matrix.triple }}" --version "$v" if [ "${{ matrix.triple }}" = "$(rustc -vV | sed -n 's/^host: //p')" ]; then t="$RUNNER_TEMP/try"; mkdir -p "$t"; tar -C "$t" -xzf "$RUNNER_TEMP/dist/wai-$v-${{ matrix.triple }}.tar.gz" b="$t/wai-$v-${{ matrix.triple }}/bin" @@ -156,7 +183,7 @@ jobs: export RUSTFLAGS="-C target-feature=+crt-static -C relocation-model=static" curl -fsS -o "$RUNNER_TEMP/web.tar.gz" "${FORGE_API%/api/v1}/$FORGE_REPO/releases/download/$GITHUB_REF_NAME/wai-web-$v.tar.gz" tar -C "$RUNNER_TEMP" -xzf "$RUNNER_TEMP/web.tar.gz" - bash wai/quickstart/build.sh "$RUNNER_TEMP/dist" --web "$RUNNER_TEMP/web" --target "$t" --version "$v" + bash wai/quickstart/build.sh "$RUNNER_TEMP/dist" --web "$RUNNER_TEMP/web" --studio "$RUNNER_TEMP/studio" --target "$t" --version "$v" d="$RUNNER_TEMP/try"; mkdir -p "$d"; tar -C "$d" -xzf "$RUNNER_TEMP/dist/wai-$v-$t.tar.gz" b="$d/wai-$v-$t/bin" (cd "$d" && "$b/wai" hello h >/dev/null && "$b/wai" verify h >/dev/null && mkdir p && cd p && "$b/wai" dev --check --port 0) @@ -194,8 +221,10 @@ jobs: @{ dir = 'wai\wai-rs'; bin = 'wai'; args = @('--no-default-features', '--features', 'quickstart', '--bin', 'wai') }, @{ dir = 'joule-code\joule-code-rs'; bin = 'wai-relay'; args = @('-p', 'joule-moq-relay', '--bin', 'wai-relay') }, @{ dir = 'joule-code\joule-code-rs\crates\joule-moq-room'; bin = 'wai-room'; args = @('--bin', 'wai-room') }, + @{ dir = 'joule-code\joule-code-rs'; bin = 'wai-ingest'; args = @('-p', 'joule-moq-ingest', '--bin', 'wai-ingest') }, @{ dir = 'joule-code\joule-code-rs\crates\joule-moq-remote'; bin = 'wai-remote-host'; args = @('--bin', 'wai-remote-host') }, - @{ dir = 'wai\wai-pack'; bin = 'wai-pack'; args = @('--bin', 'wai-pack') } + @{ dir = 'wai\wai-pack'; bin = 'wai-pack'; args = @('--bin', 'wai-pack') }, + @{ dir = 'wai\wai-quantum-cli'; bin = 'wai-quantum'; args = @('--bin', 'wai-quantum') } ) foreach ($b in $builds) { Push-Location $b.dir @@ -206,9 +235,12 @@ jobs: Pop-Location } Copy-Item -Recurse (Join-Path $env:RUNNER_TEMP 'web') (Join-Path $stage 'share\wai\web') + Copy-Item -Recurse (Join-Path $env:RUNNER_TEMP 'studio') (Join-Path $stage 'share\wai\studio') Copy-Item wai\quickstart\README.md, wai\LICENSE $stage & (Join-Path $stage 'bin\wai.exe') create --list if ($LASTEXITCODE -ne 0) { throw "the packaged wai does not run" } + & (Join-Path $stage 'bin\wai-quantum.exe') --version + if ($LASTEXITCODE -ne 0) { throw "the packaged wai-quantum does not run" } Compress-Archive -Path $stage -DestinationPath (Join-Path $dist "$name.zip") -Force $h = (Get-FileHash (Join-Path $dist "$name.zip") -Algorithm SHA256).Hash.ToLower() Set-Content -NoNewline -Path (Join-Path $dist "$name.zip.sha256") -Value "$h $name.zip`n" diff --git a/joule-code/CHANGELOG.md b/joule-code/CHANGELOG.md index bd20d16fa..433ffcb77 100644 --- a/joule-code/CHANGELOG.md +++ b/joule-code/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to the Joule Code standard are recorded here. The format fol ## [Unreleased] +### Changed — `joule-moq-room`: the example's `relay_url` is the relay's WebSocket port, 4480 + +- **`room.example.toml`** named the relay at `ws://127.0.0.1:9681/moq`, a port nothing listens on: + the relay's example, and `wai-relay --dev`, listen for WebSocket on `127.0.0.1:4480`. The two + examples now connect as shipped. `relay_url` has no default in code, so a deployment's own + configuration is unchanged. +- **`tests/ports.rs`** holds the local stack's ports in one place: the room example's `relay_url` + is the relay example's WebSocket endpoint and path, the ingest gateway example's `[relay] addr` + is its QUIC endpoint and each listener it names is a row (off until set where it is commented + out), the relay example's endpoints are `wai-relay --dev`'s, `wai dev`'s origin is the one the + remote desktop template's host reaches, and the defaults of the audit collector, the ledger, the + fleet service, the quantum job service, the worlds' relay, the stitched presentation verifier and + `wai http origin` are their sources'; each is a row of the port table in WAI's quickstart README, + and no port is two programs'. The room service started from its example opens its clock's + session on the relay started from the relay's. +- **The gateway's example is read by the gateway's own reader** (`GatewayConfig::from_toml`; + `joule-moq-ingest` is a dev-dependency of `joule-moq-room`), not as a TOML table matched by + text: its relay address and each listener's come from what the gateway reads, a commented-out + listener is one it reads as off and, uncommented, at its row's port, and a key the gateway + renamed or stopped reading fails the test. The gateway started from its example, its ports + moved to free ones and the files it names made, serves HTTP, opens RTMP, and opens none of the + listeners the example leaves off. + ### Changed — `joule-moq-ingest`: the contribution link record's magic is `WLK1` - **Wire change.** `linkrec::MAGIC` is `WLK1` (WAI `extensions/live-contribution.md` §6.2), so the link record no diff --git a/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.lock b/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.lock index c9e9d2cf0..2d982cd34 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.lock +++ b/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.lock @@ -70,7 +70,7 @@ dependencies = [ "asn1-rs-derive", "asn1-rs-impl", "displaydoc", - "nom", + "nom 7.1.3", "num-traits", "rusticata-macros", "thiserror", @@ -123,6 +123,30 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -249,13 +273,25 @@ dependencies = [ "shlex 2.0.1", ] +[[package]] +name = "ccm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae3c82e4355234767756212c570e29833699ab63e6ffd161887314cc5b43847" +dependencies = [ + "aead", + "cipher", + "ctr", + "subtle", +] + [[package]] name = "cexpr" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -368,6 +404,15 @@ dependencies = [ "libloading", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -458,6 +503,21 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -595,6 +655,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid 0.9.6", + "der_derive", + "flagset", "pem-rfc7468", "zeroize", ] @@ -617,12 +679,23 @@ checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ "asn1-rs", "displaydoc", - "nom", + "nom 7.1.3", "num-bigint", "num-traits", "rusticata-macros", ] +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "deranged" version = "0.5.8" @@ -653,6 +726,43 @@ dependencies = [ "ctutils", ] +[[package]] +name = "dimpl" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e769119713c7cf11efb6dd594258095c4a5959fa2bb8d4a7f89978efbd0c08b" +dependencies = [ + "aes", + "aes-gcm", + "arrayvec", + "aws-lc-rs", + "ccm", + "chacha20 0.9.1", + "chacha20poly1305", + "der 0.7.10", + "ecdsa", + "generic-array", + "hkdf 0.12.4", + "hmac 0.12.1", + "log", + "nom 8.0.0", + "once_cell", + "p256", + "p384", + "pkcs8 0.10.2", + "rand 0.9.5", + "rand_core 0.6.4", + "rcgen", + "sec1", + "sha2 0.10.9", + "signature 2.2.0", + "spki 0.7.3", + "subtle", + "time", + "x25519-dalek 2.0.1", + "x509-cert", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -664,6 +774,12 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "ecdsa" version = "0.16.9" @@ -744,7 +860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -793,6 +909,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "foldhash" version = "0.2.0" @@ -808,6 +930,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.34" @@ -1386,6 +1514,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "is" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6edbbdc611843e8c65878ee735f68c495cede11ae5a9582b17fe0369c456ef5" +dependencies = [ + "crc", + "serde", + "str0m-proto", + "subtle", + "tracing", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1460,6 +1601,23 @@ dependencies = [ "libc", ] +[[package]] +name = "joule-contrib" +version = "0.1.0" +dependencies = [ + "aes", + "ctr", + "hex", + "hmac 0.12.1", + "rand_core 0.6.4", + "serde", + "serde_json", + "sha1", + "sha2 0.10.9", + "tokio", + "tracing", +] + [[package]] name = "joule-moq" version = "0.3.0" @@ -1486,6 +1644,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "joule-moq-ingest" +version = "0.1.0" +dependencies = [ + "base64", + "blake3", + "bytes", + "ed25519-dalek", + "futures", + "hex", + "http-body-util", + "hyper", + "hyper-util", + "joule-contrib", + "joule-moq", + "joule-moq-relay", + "quinn", + "rand_core 0.6.4", + "rustls", + "serde", + "serde_json", + "sha2 0.10.9", + "str0m", + "thiserror", + "tokio", + "tokio-rustls", + "toml", + "tracing", + "tracing-subscriber", +] + [[package]] name = "joule-moq-mls" version = "0.1.0" @@ -1568,6 +1757,7 @@ dependencies = [ "hyper", "hyper-util", "joule-moq", + "joule-moq-ingest", "joule-moq-mls", "joule-moq-relay", "joule-moq-remote", @@ -1995,6 +2185,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2395,7 +2594,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2529,6 +2728,7 @@ version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "time", @@ -2585,7 +2785,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -2610,7 +2810,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -2623,7 +2823,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2680,7 +2880,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2697,7 +2897,7 @@ checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -2724,6 +2924,21 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sctp-proto" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ba0d5d00068c8e8ca721d55a8f14b82b768a1fe3982243c7333272dc49cb801" +dependencies = [ + "bytes", + "crc", + "log", + "rand 0.9.5", + "rustc-hash", + "slab", + "thiserror", +] + [[package]] name = "sec1" version = "0.7.3" @@ -3028,6 +3243,59 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "str0m" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e80c4a4ef148e4a9fd15b9525c7f376d97e86729f19880b591ad220f1288c0f" +dependencies = [ + "arrayvec", + "base64ct", + "combine", + "dimpl", + "fastrand", + "is", + "sctp-proto", + "serde", + "str0m-proto", + "str0m-rust-crypto", + "subtle", + "time", + "tracing", +] + +[[package]] +name = "str0m-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f3d99f2cf6c76a502e45feb896ea71e54787ede8744bcdb215acfbfcb905dd" +dependencies = [ + "base64ct", + "dimpl", + "fastrand", + "serde", + "subtle", + "time", +] + +[[package]] +name = "str0m-rust-crypto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947c2417bf43e47504911f92f47c22b6125a57d162e1661f1c77b78ebb3aa9d3" +dependencies = [ + "aes", + "aes-gcm", + "ctr", + "dimpl", + "hmac 0.12.1", + "p256", + "sha1", + "sha2 0.10.9", + "str0m-proto", + "time", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3085,10 +3353,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3402,6 +3670,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -3578,7 +3852,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3815,6 +4089,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", +] + [[package]] name = "x509-parser" version = "0.18.1" @@ -3822,10 +4107,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" dependencies = [ "asn1-rs", + "aws-lc-rs", "data-encoding", "der-parser", "lazy_static", - "nom", + "nom 7.1.3", "oid-registry", "ring", "rusticata-macros", diff --git a/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.toml b/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.toml index 332ef5f8a..8a380872a 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.toml +++ b/joule-code/joule-code-rs/crates/joule-moq-room/Cargo.toml @@ -55,3 +55,6 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros"] } # The remote desktop host agent and its headless viewer, for a remote room's session through the # service (tests/remote_room.rs). joule-moq-remote = { path = "../joule-moq-remote" } +# The ingest gateway, whose example configuration tests/ports.rs reads with the gateway's own +# reader and starts the gateway from. +joule-moq-ingest = { path = "../joule-moq-ingest" } diff --git a/joule-code/joule-code-rs/crates/joule-moq-room/room.example.toml b/joule-code/joule-code-rs/crates/joule-moq-room/room.example.toml index f4d802721..9723ef925 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-room/room.example.toml +++ b/joule-code/joule-code-rs/crates/joule-moq-room/room.example.toml @@ -8,8 +8,10 @@ listen = "127.0.0.1:9682" # over HTTP, and a watch room's join proof, name the path the page addressed. public_prefix = "/room-api" keys = "/path/to/keys.toml" -# The relay's WebSocket endpoint, for the clock's own session (loopback). -relay_url = "ws://127.0.0.1:9681/moq" +# The relay's WebSocket endpoint, for the clock's own session (loopback): the relay's +# `websocket_listen` and `path`, as relay.example.toml sets them (tests/ports.rs holds the two +# files, and the port table in wai/quickstart/README.md, to each other). +relay_url = "ws://127.0.0.1:4480/moq" # Exit with status 70 past this resident memory (MiB), for the supervisor to restart it. max_resident_mb = 256 diff --git a/joule-code/joule-code-rs/crates/joule-moq-room/src/lib.rs b/joule-code/joule-code-rs/crates/joule-moq-room/src/lib.rs index 63907b196..2b97b83b7 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-room/src/lib.rs +++ b/joule-code/joule-code-rs/crates/joule-moq-room/src/lib.rs @@ -1010,7 +1010,7 @@ mod tests { assert_eq!((c.co.lead, c.co.ai, c.public_prefix.as_str()), (Lead::First, true, "/room-api")); let on = Config::from_toml(&format!("{}\n", include_str!("../room.example.toml").replace("audience_input = 0", "audience_input = 3"))).unwrap(); assert!(on.settings().unwrap().co()); - assert_eq!(proof_target("ws://127.0.0.1:9681/moq"), "//127.0.0.1:9681/moq"); + assert_eq!(proof_target(&c.relay_url), "//127.0.0.1:4480/moq"); assert_eq!(proof_target("wss://relay.example/moq?x=1"), "//relay.example/moq"); assert!(Config::from_toml("listen = \"127.0.0.1:1\"\nkeys = \"k\"\nrelay_url = \"ws://x\"\n[co]\nlead = \"everyone\"\n").is_err(), "a lead the service does not know"); } diff --git a/joule-code/joule-code-rs/crates/joule-moq-room/tests/ports.rs b/joule-code/joule-code-rs/crates/joule-moq-room/tests/ports.rs new file mode 100644 index 000000000..6d6ce54de --- /dev/null +++ b/joule-code/joule-code-rs/crates/joule-moq-room/tests/ports.rs @@ -0,0 +1,267 @@ +//! The local stack's ports, held in one place. Every default port of every program, and every +//! port one program's example configuration names to reach another, agrees with the program that +//! serves it, and with the port table of wai/quickstart/README.md: +//! +//! - wai-relay: relay.example.toml's endpoints are `wai-relay --dev`'s (`RelayConfig::dev`); +//! - wai-room: room.example.toml's `relay_url` is the relay example's WebSocket endpoint and path; +//! - wai-ingest: gateway.example.toml, read by the gateway's own reader (`GatewayConfig`): its +//! `[relay] addr` is the relay example's QUIC endpoint, and each listener it names is a row, +//! those it leaves commented out (off until set) said to be, each of those read as the gateway +//! reads it once its line is uncommented; +//! - `wai dev`'s origin, which the remote desktop template's host.sh reaches by default; +//! - every other daemon whose default is in its source: the audit collector, the ledger, the +//! fleet service, the quantum job service, the worlds' relay, the stitched presentation +//! verifier and `wai http origin`; +//! +//! no two defaults are one port; the room service started from its example file opens its +//! clock's session on the relay started from the relay's, each moved to a free port alike; and +//! the gateway started from its example, its ports moved to free ones and its files made, opens +//! the listeners the table says are on and none of those it says are off. + +use joule_moq_ingest::{GatewayConfig, GatewayServer}; +use joule_moq_relay::{RelayConfig, RelayServer}; +use joule_moq_room::keys::Keys; +use joule_moq_room::{Config, Service}; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; +use std::time::Duration; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../..") +} + +fn read(rel: &str) -> String { + std::fs::read_to_string(repo().join(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")) +} + +const ROOM_EXAMPLE: &str = "joule-code/joule-code-rs/crates/joule-moq-room/room.example.toml"; +const RELAY_EXAMPLE: &str = "joule-code/joule-code-rs/crates/joule-moq-relay/relay.example.toml"; +const GATEWAY_EXAMPLE: &str = "joule-code/joule-code-rs/crates/joule-moq-ingest/gateway.example.toml"; +const TABLE: &str = "wai/quickstart/README.md"; + +/// `ws://host:port/path` as (scheme, authority, path). +fn split_url(u: &str) -> (&str, &str, &str) { + let (scheme, rest) = u.split_once("://").unwrap_or_else(|| panic!("{u}: a URL")); + let (authority, path) = rest.find('/').map_or((rest, ""), |i| (&rest[..i], &rest[i..])); + (scheme, authority, path) +} + +/// A row of the port table: its program, what listens there, and the setting that moves it. +struct Row { + program: String, + what: String, + setting: String, +} + +/// What a row says of a listener that is off until its setting is given. +const OFF: &str = "off until set"; + +/// The port table: each row's port (with `/udp` for a UDP one) and the row. +fn table() -> BTreeMap { + let text = read(TABLE); + let heading = "\n## Ports\n"; + let at = text.find(heading).expect("the README has a port table") + heading.len(); + let mut rows = BTreeMap::new(); + for line in text[at..].lines().take_while(|l| !l.starts_with("## ")) { + let cells: Vec<&str> = line.split('|').map(str::trim).collect(); + if cells.len() < 6 || !cells[1].starts_with(|c: char| c.is_ascii_digit()) { + continue; + } + let unquote = |s: &str| s.trim_matches('`').to_string(); + let row = Row { program: unquote(cells[2]), what: cells[3].to_string(), setting: unquote(cells[4]) }; + assert!(rows.insert(cells[1].to_string(), row).is_none(), "{}: twice in the table", cells[1]); + } + rows +} + +/// `text` with the line that gives `key` in its table `[section]` uncommented, when the example +/// leaves it commented out (a listener the program opens only once it is set); `None` when the +/// line is not commented out. +fn uncommented(text: &str, section: &str, key: &str) -> Option { + let header = format!("[{section}]"); + let lines: Vec<&str> = text.lines().collect(); + let at = lines.iter().position(|l| l.trim() == header).unwrap_or_else(|| panic!("no {header}")); + for (i, line) in lines.iter().enumerate().skip(at + 1).take_while(|(_, l)| !l.trim_start().starts_with('[')) { + let (off, body) = match line.trim_start().strip_prefix('#') { + Some(rest) => (true, rest.trim_start()), + None => (false, line.trim_start()), + }; + if body.strip_prefix(key).is_some_and(|r| r.trim_start().starts_with('=')) { + if !off { + return None; + } + let mut out: Vec = lines.iter().map(|l| l.to_string()).collect(); + out[i] = body.to_string(); + return Some(out.join("\n")); + } + } + panic!("{header} names no {key}") +} + +/// The gateway's listeners as its reader reads them, each with its section, its key, whether it +/// is UDP, and its address in a configuration. +type Listener = (&'static str, &'static str, bool, fn(&GatewayConfig) -> Option); +const GATEWAY_LISTENERS: &[Listener] = &[ + ("http", "listen", false, |c| Some(c.http.listen)), + ("rtmp", "listen", false, |c| c.rtmp.listen), + ("rtmp", "tls_listen", false, |c| c.rtmp.tls_listen), + ("injector", "listen", false, |c| c.injector.listen), + ("contrib.srt", "listen", true, |c| c.contrib.srt.listen), +]; + +/// The numbers of four or five digits on a line of source: the ports it names. +fn ports_on(line: &str) -> Vec { + line.split(|c: char| !c.is_ascii_digit()).filter(|n| (4..=5).contains(&n.len())).filter_map(|n| n.parse().ok()).collect() +} + +#[test] +fn every_default_port_agrees_with_the_program_that_serves_it() { + let t = table(); + let mut held = Vec::new(); + let mut row = |port: String, program: &str, setting: &str, off: bool| { + let r = t.get(&port).unwrap_or_else(|| panic!("the table has no row for {port} ({program})")); + assert_eq!((r.program.as_str(), r.setting.as_str()), (program, setting), "the table's row for {port}"); + assert_eq!(r.what.contains(OFF), off, "the table's row for {port} says {OFF:?} exactly when the listener is off until set"); + held.push(port); + }; + + // The relay: its example's endpoints are its development defaults, and the table's. + let relay = RelayConfig::from_toml(&read(RELAY_EXAMPLE)).expect("the relay's example reads"); + let dev = RelayConfig::dev(); + assert_eq!((relay.quic_listen, relay.websocket_listen, relay.path.as_str()), (dev.quic_listen, dev.websocket_listen, dev.path.as_str()), "{RELAY_EXAMPLE} is `wai-relay --dev`"); + let quic = relay.quic_listen.expect("a QUIC endpoint"); + let ws = relay.websocket_listen.expect("a WebSocket endpoint"); + row(format!("{}/udp", quic.port()), "wai-relay", "quic_listen", false); + row(ws.port().to_string(), "wai-relay", "websocket_listen", false); + + // The room service: its relay is the relay's WebSocket endpoint, at the relay's path. + let room = Config::from_toml(&read(ROOM_EXAMPLE)).expect("the room's example reads"); + let (scheme, authority, path) = split_url(&room.relay_url); + let at: SocketAddr = authority.parse().unwrap_or_else(|e| panic!("{authority}: {e}")); + assert_eq!((scheme, at, path), ("ws", ws, relay.path.as_str()), "{ROOM_EXAMPLE}'s relay_url is the relay's WebSocket endpoint"); + row(room.listen.port().to_string(), "wai-room", "listen", false); + + // The ingest gateway, its example read by its own reader: its relay is the relay's QUIC + // endpoint, and each listener its example names is a row. One the example leaves commented + // out is off (the reader reads none) and its row says so; uncommented, the reader reads it at + // the row's port. + let text = read(GATEWAY_EXAMPLE); + let gw = GatewayConfig::from_toml(&text).unwrap_or_else(|e| panic!("{GATEWAY_EXAMPLE} reads as the gateway reads it: {e}")); + assert_eq!(gw.relay.addr, quic, "{GATEWAY_EXAMPLE}'s [relay] addr is the relay's QUIC endpoint"); + for (section, key, udp, at) in GATEWAY_LISTENERS { + let (a, off) = match uncommented(&text, section, key) { + None => (at(&gw).unwrap_or_else(|| panic!("{GATEWAY_EXAMPLE}: [{section}] {key} is set, and the gateway reads none")), false), + Some(on) => { + assert_eq!(at(&gw), None, "{GATEWAY_EXAMPLE}: [{section}] {key} is commented out, and the gateway reads one"); + let c = GatewayConfig::from_toml(&on).unwrap_or_else(|e| panic!("{GATEWAY_EXAMPLE} with [{section}] {key} set: {e}")); + (at(&c).unwrap_or_else(|| panic!("{GATEWAY_EXAMPLE}: [{section}] {key}, uncommented, is not what the gateway reads")), true) + } + }; + let port = format!("{}{}", a.port(), if *udp { "/udp" } else { "" }); + row(port, "wai-ingest", &format!("[{section}] {key}"), off); + } + + // `wai dev`'s origin: its default, and the one the remote desktop template's host reaches. + let dev_rs = read("wai/wai-rs/src/bin/wai_cli/dx/dev.rs"); + let origin = t.iter().find(|(_, r)| r.program == "wai dev").map(|(port, _)| port.clone()).expect("a row for wai dev"); + assert!(dev_rs.contains(&format!("None => {origin},")), "wai dev's default port is {origin}"); + row(origin.clone(), "wai dev", "--port", false); + assert!(read("wai/examples/remote-desktop/host.sh").contains(&format!("WAI_DEV_ORIGIN:-http://127.0.0.1:{origin}}}")), "host.sh reaches wai dev's origin"); + + // The daemons whose defaults are in their sources: each source, the text on the line that + // reads the setting, the program and the setting. That line names one port, the table's. + let sources = [ + ("wai/wai-audit/src/main.rs", "env_or(\"WAI_AUDIT_ADDR\"", "wai-audit", "WAI_AUDIT_ADDR"), + ("wai/wai-ledger/src/main.rs", "var(\"WAI_LEDGER_ADDR\")", "wai-ledger", "WAI_LEDGER_ADDR"), + ("wai/wai-fleet/src/main.rs", "var(\"WAI_FLEET_ADDR\")", "wai-fleet", "WAI_FLEET_ADDR"), + ("wai/wai-quantum-service/src/main.rs", "var(\"WAI_QSVC_ADDR\")", "wai-quantum-service", "WAI_QSVC_ADDR"), + ("wai/wai-relay/src/main.rs", "var(\"WAI_RELAY_BIND\")", "wai-world-relay", "WAI_RELAY_BIND"), + ("wai/wai-rs/src/bin/wai_stitched_presentation.rs", "opt(a, \"--listen\")", "wai_stitched_presentation serve", "--listen"), + ("wai/wai-rs/src/bin/wai_cli/http.rs", "opt(args, \"--port\").map_or(", "wai http origin", "--port"), + ]; + for (src, reads, program, setting) in sources { + let text = read(src); + let lines: Vec<&str> = text.lines().filter(|l| l.contains(reads) && !l.trim_start().starts_with("//")).collect(); + assert_eq!(lines.len(), 1, "{src}: one line reads {setting}: {lines:?}"); + let ports = ports_on(lines[0]); + assert_eq!(ports.len(), 1, "{src}: the line that reads {setting} names one port: {}", lines[0]); + row(ports[0].to_string(), program, setting, false); + } + + // Every row is one held above; the table holds each port once, so no two programs' defaults + // are one port on one protocol. + held.sort(); + assert_eq!(held, t.keys().cloned().collect::>(), "the rows held and the table's"); +} + +/// The room service, started from room.example.toml, opens its clock's session on the relay +/// started from relay.example.toml. Both move to free ports alike (the room's `relay_url` keeps +/// its scheme, host and path, its port the relay's), and the relay keeps nothing on disk. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_room_example_reaches_the_relay_example() { + let mut rc = RelayConfig::from_toml(&read(RELAY_EXAMPLE)).unwrap(); + rc.quic_listen = None; + rc.websocket_listen = Some("127.0.0.1:0".parse().unwrap()); + rc.dvr = None; + let relay = RelayServer::start(rc).await.expect("the relay starts from its example"); + let ws = relay.websocket_addr.expect("its WebSocket endpoint"); + + let text = read(ROOM_EXAMPLE); + let example = Config::from_toml(&text).unwrap(); + let (scheme, authority, path) = split_url(&example.relay_url); + let moved = format!("{scheme}://{}:{}{path}", authority.rsplit_once(':').unwrap().0, ws.port()); + let text = text + .replace(&format!("relay_url = \"{}\"", example.relay_url), &format!("relay_url = \"{moved}\"")) + .replace(&format!("listen = \"{}\"", example.listen), "listen = \"127.0.0.1:0\""); + let cfg = Config::from_toml(&text).unwrap(); + assert_eq!(cfg.relay_url, moved, "the example's relay_url, moved"); + let svc = Service::new(Keys::generate(), &cfg); + tokio::spawn(svc.clone().run_clock()); + let up = tokio::time::timeout(Duration::from_secs(30), async { + while !svc.connected.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await; + assert!(up.is_ok(), "the clock's session opens on the relay at {moved}"); +} + +/// The gateway, started from gateway.example.toml with its ports moved to free ones and the +/// files it names made (its signing key, the relay's certificate and token, a room's signed +/// descriptor): it serves HTTP, opens RTMP, which the example sets, and none of the listeners +/// the example leaves off (RTMPS, the SCTE 104 injector, SRT). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_gateway_starts_from_its_example() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let dir = std::env::temp_dir().join(format!("ports-gateway-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let keys = Keys::generate(); + let now_us = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_micros() as u64; + let room = joule_moq_room::epoch::descriptor(&keys, joule_moq_room::epoch::roles(false, 0, false), 8, 4, now_us); + let mut text = read(GATEWAY_EXAMPLE); + let example = GatewayConfig::from_toml(&text).unwrap(); + for (name, bytes) in [("gateway.key", vec![7u8; 32]), ("relay-cert.pem", Vec::new()), ("relay-token", b"token".to_vec()), ("room.wrd1", room.to_bytes())] { + std::fs::write(dir.join(name), bytes).unwrap(); + let from = format!("\"{name}\""); + assert_eq!(text.matches(&from).count(), 1, "{GATEWAY_EXAMPLE} names {name} once"); + text = text.replace(&from, &format!("{:?}", dir.join(name).to_str().unwrap())); + } + for at in [Some(example.http.listen), example.rtmp.listen] { + let from = format!("listen = \"{}\"", at.unwrap()); + assert_eq!(text.matches(&from).count(), 1, "{GATEWAY_EXAMPLE}: {from} once"); + text = text.replace(&from, "listen = \"127.0.0.1:0\""); + } + let cfg = GatewayConfig::from_toml(&text).unwrap(); + let gw = GatewayServer::start(cfg).await.unwrap_or_else(|e| panic!("the gateway starts from its example: {e}")); + assert!(gw.addr.ip().is_loopback() && gw.rtmp_addr.is_some_and(|a| a.ip().is_loopback()), "HTTP and RTMP, on loopback"); + assert_eq!((gw.rtmps_addr, gw.injector_addr, gw.srt_addr), (None, None, None), "RTMPS, the injector and SRT, off until set"); + assert!(gw.gateway.rooms.contains_key(&room.room_id()), "the room its example names"); + let mut s = tokio::net::TcpStream::connect(gw.addr).await.unwrap(); + s.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n").await.unwrap(); + let mut head = [0u8; 9]; + s.read_exact(&mut head).await.unwrap(); + assert_eq!(&head, b"HTTP/1.1 ", "the gateway answers HTTP at its listener"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/scripts/check-wai-ops-doors-selftest.sh b/scripts/check-wai-ops-doors-selftest.sh index 883cabaae..5fd1af386 100755 --- a/scripts/check-wai-ops-doors-selftest.sh +++ b/scripts/check-wai-ops-doors-selftest.sh @@ -141,6 +141,17 @@ expect fail "a C function misnamed" "wai_envelope_decode names envelope.decode, mk; printf 'int32_t wai_receipt_get(const uint8_t *p);\n' >> "$tmp/wai/wai-rs/include/wai.h" expect fail "a C function with a near-synonym verb" "wai_receipt_get: the verb get is a near-synonym; the operation verb is show" +mk; printf 'int32_t wai_room_up(const uint8_t *p);\n' >> "$tmp/wai/wai-rs/include/wai.h" +expect fail "a C function whose verb is up, start's near-synonym" "wai_room_up: the verb up is a near-synonym; the operation verb is start" + +# upscale and tonemap joined N3's list after its approval: a door's name with either verb is an +# operation's name, held like any other. +mk; printf 'int32_t wai_envelope_upscale(const uint8_t *p);\n' >> "$tmp/wai/wai-rs/include/wai.h" +expect fail "a C function whose verb is upscale" "wai_envelope_upscale names envelope.upscale, which the catalogue does not hold" + +mk; sub "$tmp/wai/wai-sink-swift/Sources/WaiSink/Envelope.swift" 'public func pack() {}' 'public func tonemap() {}' +expect fail "a Swift method whose verb is tonemap" "WaiEnvelope.tonemap names envelope.tonemap, which the catalogue does not hold" + mk; sub "$tmp/wai/wai-sink-swift/Sources/WaiSink/Envelope.swift" 'public func pack() {}' 'public func verify() {}' expect fail "a Swift method misnamed" "WaiEnvelope.verify names envelope.verify, which the catalogue does not hold" diff --git a/scripts/check-wai-ops-doors.py b/scripts/check-wai-ops-doors.py index c062df8f8..bf8b75dba 100755 --- a/scripts/check-wai-ops-doors.py +++ b/scripts/check-wai-ops-doors.py @@ -20,7 +20,7 @@ What each door is held to: (placement `service` or `both`), at the route the catalogue gives it. - C, Swift, Kotlin: a name in a catalogue noun's namespace whose verb is an operation verb (the closed list below) names an operation, and that operation must be in the catalogue; - a near-synonym verb (`get`, `describe`, `info`, `keygen`, `init`, `new`) in a catalogue + a near-synonym verb (`get`, `describe`, `info`, `keygen`, `init`, `new`, `up`) in a catalogue noun's namespace is refused. Any other name is library API the catalogue does not list (the C ABI's codec and sink functions, the native sinks' classes), and is reported as such. @@ -37,15 +37,19 @@ import os import re import sys -# The operation verbs: the closed list of WAI toolchain v1 (naming rule N3). +# The operation verbs: the closed list of WAI toolchain v1 (naming rule N3), with the two it +# gained after its approval, upscale and tonemap (wai/catalogue/README.md). The catalogue's +# own copy is wai/wai-rs/src/catalogue.rs's VERBS; a test there holds this one to it. VERBS = { "create", "list", "show", "update", "delete", "check", "verify", "run", "build", "export", "import", "start", "stop", "join", "leave", "grant", "invite", "revoke", "rotate", "tail", "search", "test", "load", "encode", "decode", "inspect", "wrap", "unwrap", "fetch", "generate", "promote", "rollback", "lock", "upgrade", "setup", "guide", "stream", + "upscale", "tonemap", } -# Near-synonyms of an operation verb, and the verb to use. -NEAR = {"get": "show", "describe": "show", "info": "show", "keygen": "create", "init": "create", "new": "create"} +# Near-synonyms of an operation verb, and the verb to use: the catalogue's NEAR_SYNONYMS +# (wai/wai-rs/src/catalogue.rs), which a test there holds this one to. +NEAR = {"get": "show", "describe": "show", "info": "show", "keygen": "create", "init": "create", "new": "create", "up": "start"} MCP_TABLE = "wai/wai-rs/src/bin/wai_cli/mcp/tools.rs" OPENAPI_GLOBS = ["joule-code/joule-code-rs/crates/*/openapi.json", "wai/*/openapi.json", "wai/wai-rs/openapi.json"] diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 7cef39f74..257c43ad3 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -311,6 +311,9 @@ wire format may change and such changes are recorded here. - **The corpus** [`live-conformance/holdings/`](live-conformance/holdings/): 21 records (a control for each rule and for the signature, a small-order key among them), 11 verdicts, 6 queries and a plan over three neighbours. `holdings_verify.py` is a second implementation. +- **The registry** lists `WHD1` as a magic of holdings §4.1 (REGISTRY.md and `codecs/registry.rs`), a magic no + other record, wire identifier, open branch or crate of the repository uses; `holdings::MAGIC` is held to that one + row. ### When each object passes on a bearer with no return path, and when a receiver holds it diff --git a/wai/REGISTRY.md b/wai/REGISTRY.md index dee3f8f18..e62a50f2f 100644 --- a/wai/REGISTRY.md +++ b/wai/REGISTRY.md @@ -403,6 +403,7 @@ and a role are fixed once registered. | `WFE1` | magic | defined | fleet §3.1 | | `WFR1` | magic | defined | fleet §6.1 | | `WGM1` | magic | defined | exact-tonemap §4.1 | +| `WHD1` | magic | defined | holdings §4.1 | | `WHP1` | magic | defined | http-delivery §6.2 | | `WJR1` | magic | defined | jwp-receipts §2.7.3 | | `WKE1` | magic | defined | key-release §5.1 | diff --git a/wai/catalogue/README.md b/wai/catalogue/README.md index 72c805c77..8a6c334be 100644 --- a/wai/catalogue/README.md +++ b/wai/catalogue/README.md @@ -15,9 +15,19 @@ rule N3's closed list (create, list, show, update, delete, check, verify, run, b import, start, stop, join, leave, grant, invite, revoke, rotate, tail, search, test, load, encode, decode, inspect, wrap, unwrap, fetch, generate, promote, rollback, lock, upgrade, setup, guide, stream, upscale, tonemap), never a near-synonym (`get`, `describe` and `info` are `show`; `keygen`, `init` and -`new` are `create`). A command a person already types keeps its spelling as the operation's -command line: `wai hello` is `toolchain.test`, `wai stage refine` is `stage.encode`. Its entry -(format `wai.ops/1`) carries: +`new` are `create`; `up` is `start`, as in `wai dev`). A command a person already types keeps its spelling as the operation's +command line: `wai hello` is `toolchain.test`, `wai stage refine` is `stage.encode`. + +The list as approved with WAI toolchain v1 ended at `stream`. Two verbs joined it after the wai +0.4.0 release, with `media.upscale` and `media.tonemap`: `upscale` and `tonemap`. Each names an +exact derivation, an output any verifier recomputes bit for bit from the input and the parameters +its record names (extensions/exact-upscale.md, extensions/exact-tonemap.md), and no approved verb +names that: `encode` and `decode` run a codec, and `generate` makes media from a description. A +verb joins the list the same way: in `VERBS` (wai-rs/src/catalogue.rs), this paragraph and the +list above, and the doors' check (`scripts/check-wai-ops-doors.py`), which a test of the catalogue +holds to `VERBS`. + +An entry (format `wai.ops/1`) carries: | Member | Meaning | |---|---| diff --git a/wai/changes/2026-11-04-every-template-carries-agents-md.md b/wai/changes/2026-11-04-every-template-carries-agents-md.md new file mode 100644 index 000000000..1cac74746 --- /dev/null +++ b/wai/changes/2026-11-04-every-template-carries-agents-md.md @@ -0,0 +1,12 @@ +### Every template carries an AGENTS.md + +- **The nine templates of `wai create`** (the examples in [examples/](examples/)) each carry an + `AGENTS.md`, the instructions a coding agent reads: what the project is, and how to build it, + run it, test it (the command that exits non-zero when it stops working: `wai dev --check` for a + room, the verifiers' verdicts, a replay through the native sink, a synthetic stream) and conform + it, with what never to do to make a check pass, and that no secret belongs in the repository. + `wai create` writes it with the rest, and `wai setup-agent` adds its own block to it. An app + exported from Create already carried one. +- **The remote desktop template's** `.gitignore` keeps out `host.key`, the host's private key that + `host.sh` makes in the project. +- A test of `wai create` holds every template to carrying one, with each of those sections. diff --git a/wai/changes/2026-11-04-local-stack-ports-one-table.md b/wai/changes/2026-11-04-local-stack-ports-one-table.md new file mode 100644 index 000000000..506228017 --- /dev/null +++ b/wai/changes/2026-11-04-local-stack-ports-one-table.md @@ -0,0 +1,10 @@ +### The local stack's ports, in one table + +- **The quickstart's README** ([quickstart/README.md](quickstart/README.md)) has a port table: + what `wai dev`, the relay, the room service, the ingest gateway (with the listeners it opens + only once they are set), the audit collector, the ledger, the fleet service, the quantum job + service, the worlds' relay, the stitched presentation verifier and `wai http origin` listen on + by default, and which setting moves each; no two share a port. joule-moq-room's `tests/ports.rs` + holds every row to the program that serves it and to the example configurations that reach it. +- **The room service's example** names the relay's WebSocket port, 4480, where it named 9681, on + which nothing listens (joule-code's changelog). diff --git a/wai/changes/2026-11-04-quickstart-archive-carries-wai-studio-wai.md b/wai/changes/2026-11-04-quickstart-archive-carries-wai-studio-wai.md new file mode 100644 index 000000000..e519fac52 --- /dev/null +++ b/wai/changes/2026-11-04-quickstart-archive-carries-wai-studio-wai.md @@ -0,0 +1,27 @@ +### The quickstart archive carries WAI Studio, `wai-ingest` and `wai-quantum` + +- **`share/wai/studio/`**: WAI Studio, built once by the new `quickstart/build-studio.sh` (Studio's + core and Quantum Studio's engine through wasm-bindgen's web target, then both interfaces), so + `wai dev` from an installed archive serves it at `/_wai/`, where it looks beside its own binary. + Before this the archive did not carry it, and `/_wai/` answered that Studio's files were not + there. `build.sh` takes it as `--studio `, beside `--web`. +- **`bin/wai-ingest`**, the ingest gateway (WHIP, SRT and RTMP into a room, WHEP out), and + **`bin/wai-quantum`**, the quantum command line, join the five binaries; `build.sh` runs each of + them before it packages them, where it can. +- **The release** (`wai-release.yml`) builds the room client and Studio once, on the macOS host, + into one attachment, and every platform's archive carries both and the seven binaries; the + Homebrew formula installs `share/wai/studio` too, and the Scoop manifest names `wai-ingest`. +- **`wai-quantum` stays its own package.** The Homebrew formula and the Scoop manifest install + every binary of the archive but `wai-quantum`, and depend on its own formula and manifest + instead, so `brew install` and `scoop install` of either, after the other, find one copy, not + two packages claiming one command. The archive's `wai-quantum` is built as its own release + builds it (on macOS with `metal`, the state vector on the GPU), so the one `install.sh` puts on + `PATH` is the same program. `quickstart/test/formulae_test.py`, in the examples workflow, + runs `formulae.py` against a stand-in release and holds both entries to `build.sh`. +- **The checks.** `quickstart/test/clean-machine.sh` refuses an archive that lacks any of the + seven binaries, the room client, or Studio's page and core, runs `wai-quantum` and `wai-ingest` + from it, and requires `wai dev --check` to have served Studio at `/_wai/`; the examples + workflow's archive job builds Studio as the release does, requires the same of the archive's + `wai dev --check`, and runs `clean-machine.sh` on the archive it built. That job also fails + when the builds leave a file in the checkout: Studio's build, without `CARGO_TARGET_DIR`, goes + under the ignored `target/studio/`. diff --git a/wai/changes/2026-11-04-verbs-upscale-tonemap-recorded-naming-rule.md b/wai/changes/2026-11-04-verbs-upscale-tonemap-recorded-naming-rule.md new file mode 100644 index 000000000..76bbaba8b --- /dev/null +++ b/wai/changes/2026-11-04-verbs-upscale-tonemap-recorded-naming-rule.md @@ -0,0 +1,14 @@ +### The verbs `upscale` and `tonemap` are recorded in naming rule N3's list, and held in every text that states it + +- **The record.** `upscale` and `tonemap` joined the closed verb list with `media.upscale` and + `media.tonemap`, after the list was approved with WAI toolchain v1. The catalogue's README + ([catalogue/README.md](catalogue/README.md)) now says so and why: each names an exact derivation + that a verifier recomputes from what its record names, which no approved verb names. +- **The doors' check** (`scripts/check-wai-ops-doors.py`) gains both verbs. Without them a C + function, a Swift method or a Kotlin method named with either verb was taken for library API and + never held to the catalogue; its self-test now fails a C function and a Swift method that name an + operation with them that the catalogue does not hold. +- **One list.** A test of the catalogue holds the README's list and the check's to `VERBS`, and + the near-synonyms the two state beside it to `NEAR_SYNONYMS`, each with its verb. The check's + had left out `up` (`start`, as in `wai dev`), and the README's too, so a door named with it was + never refused; both now name it. diff --git a/wai/changes/2026-11-04-wai-json-names-schema-version-project.md b/wai/changes/2026-11-04-wai-json-names-schema-version-project.md new file mode 100644 index 000000000..72ed26a54 --- /dev/null +++ b/wai/changes/2026-11-04-wai-json-names-schema-version-project.md @@ -0,0 +1,33 @@ +### `wai.json` names its schema, its version and its project, and the schema is committed + +- **Three members.** `$schema`, the URL of the file's JSON Schema + (`https://wai.transaction.science/schema/wai.json/1`); `wai`, the file's version, `1`; and + `name`, the project's machine name, the name it is deployed by: a DNS label (lowercase letters, + digits and hyphens, at most 63, beginning with a letter and not ending with a hyphen). The + reader refuses a file of a later version by name (`upgrade wai`) rather than reading half of + it: by its `wai` or by its `$schema`, before any other member is read, so a member the later + version adds is never refused as unknown in its place. It refuses a `$schema` of another + schema, and a name that breaks the rule. A file without them reads as before, as version 1. +- **The schema**, [schema/wai-json-1.schema.json](schema/wai-json-1.schema.json) (JSON Schema + 2020-12), states every member and every rule the reader holds. The reader's tests hold the two + together: the same members, modes and name rule, and the same verdict on every template's file + and on files each breaking one rule, with the schema read as a validator reads it: its name + pattern run as a regular expression, and numbers compared by value. So the reader takes a + whole number however it is written (`4.0` and `4e0` are `4`, as the schema's `integer` and + `const` take them), and none above 2^53 - 1, the largest every JSON reader holds exactly, + which the schema states as `dev.watch_ms`'s maximum. +- **Served before it is named.** The site is to serve the schema at its `$id`, and the release + waits for it: the tag build's first job (`wai-release.yml`) fetches the URL and fails unless + it answers with this schema, so no release writes a `$schema` that editors and agents cannot + fetch. Until the site serves it, a tag of `wai` does not build. +- **Version 1 is fixed by the first release that writes it**, and the members the toolchain's + design adds beside these (`requires`, `room`, `package`, `conform`, `breaking`, `targets`, + `receipts` and `dev.port`) are version 2's, each with the operation that reads it. Adding them + to version 1 after that release would make every reader of it refuse such a file as having an + unknown member, unable to say `upgrade wai`; adding them now, with nothing to read them, would + make the strict reader accept members it ignores. [schema/README.md](schema/README.md) says + so. +- **`wai create`** writes the three members, and names the project for its directory when that is + a project name (`wai create my-room --template video-room` makes `my-room`), else for its + template; the rest of the template's file is unchanged. Every template's `wai.json` carries + them, named for the template. An app exported from Create carries them too. diff --git a/wai/changes/2026-11-04-wai-mcp-speaks-mcp-2026-07.md b/wai/changes/2026-11-04-wai-mcp-speaks-mcp-2026-07.md new file mode 100644 index 000000000..47f4bbac3 --- /dev/null +++ b/wai/changes/2026-11-04-wai-mcp-speaks-mcp-2026-07.md @@ -0,0 +1,26 @@ +### `wai mcp` speaks MCP 2026-07-28, the stateless revision, beside the four it negotiates + +- **No handshake.** A request whose `_meta` names its revision (`io.modelcontextprotocol/protocolVersion`) + is served from what it carries alone, as MCP 2026-07-28 serves every request: its + `io.modelcontextprotocol/clientCapabilities` is required (`-32602` without it), and a revision + the server does not serve per request is refused `-32022` (`UnsupportedProtocolVersion`) with + `data.supported`, the revisions a request may name, `data.requested`, and `data.initialize`, + the revisions `initialize` negotiates. `supported` never holds the revision asked for, so a + client that retries with one of them, as that error asks, is served rather than refused + again. A revision `initialize` negotiates (2025-11-25, 2025-06-18, 2025-03-26, 2024-11-05) + named per request is refused so, and says so. +- **`server/discover`**, which that revision requires: the revisions the server speaks, newest + first, its capabilities (`tools`) and its instructions. +- **Every result** of that revision carries `resultType` (`"complete"`) and the server's name and + version in `_meta` (`io.modelcontextprotocol/serverInfo`); `server/discover` and `tools/list` + carry `ttlMs` (an hour: what they state is fixed for the process's life) and `cacheScope` + (`"public"`: nothing in them is a user's). `ping`, which that revision removed, is `-32601` + under it. +- **The earlier revisions.** `initialize` opens a session in 2025-11-25, the last revision with + a handshake, which it now negotiates (a client of it was moved down to 2025-06-18, and + `server/discover`'s list skipped it): nothing these tools use changed in it. It still opens one + in 2025-06-18, 2025-03-26 or 2024-11-05, and a request without the revision in its `_meta` is + served under them; one process serves both, so a client of either era, or one that probes with + `server/discover` and falls back to `initialize`, works with it. +- **The tools are the same in both**: the same list, the same calls and the same results, a + tool's own error a result with `isError`. diff --git a/wai/examples/README.md b/wai/examples/README.md index 1959e4d02..888bb1f0d 100644 --- a/wai/examples/README.md +++ b/wai/examples/README.md @@ -24,6 +24,9 @@ wai dev Every example is Apache-2.0, and carries the license's text as `LICENSE`, which a project `wai create` makes from it receives with the rest. +Every example carries an `AGENTS.md`, the instructions a coding agent reads: what the project is, +and how to build, run, test and conform it. `wai setup-agent` adds WAI's own block to it. + Each example's `test/` directory holds the check that it works; it is not part of the project `wai create` writes. `bash ../quickstart/test/run-examples.sh portable|browser|apple|android` runs them, as `.github/workflows/wai-examples.yml` does. diff --git a/wai/examples/android-app/AGENTS.md b/wai/examples/android-app/AGENTS.md new file mode 100644 index 000000000..bc9474c6d --- /dev/null +++ b/wai/examples/android-app/AGENTS.md @@ -0,0 +1,21 @@ +# Working on this project + +This is a WAI project made from the `android-app` template: the WAI native sink's Kotlin API, the +one an Android app's player session calls, replaying a recorded playback session +(`sample-session.json`) through the sink on the host and verifying its bundle +(`replay/src/main/kotlin/Replay.kt`). + +- **Build.** The sink is the `wai-sink-android` project of the WAI repository: build its host + library once (`scripts/build-native.sh host` there, with JDK 17 and Rust), set + `WAI_SINK_ANDROID` to that project's directory, then `gradle :replay:build` (Gradle 8.13, as the + SDK pins it). +- **Run.** `gradle :replay:run`. +- **Test.** `gradle :replay:run` prints how many playlists and objects went through the sink and + exits 0 when the bundle verifies, non-zero otherwise. Run it after every change. +- **Conform.** In an app, a `WaiPlayerSession(context, url, config, WaiViewAttention(view))` makes + the same calls on the path of the player's requests; the SDK's README covers the AAR. Never + bypass the sink's checks to make a stream play. +- `wai.json` is the project file; its `$schema` names the schema editors validate it with. Keep + `name` a lowercase DNS label. +- Never put a key, a token or any other secret in this repository. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/android-app/wai.json b/wai/examples/android-app/wai.json index 94b492067..b0f147b75 100644 --- a/wai/examples/android-app/wai.json +++ b/wai/examples/android-app/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "android-app", "template": "android-app", "title": "The native sink in Kotlin, for Android", "summary": "The WAI native sink's Kotlin API, the one an Android app's player session calls, replaying a recorded session and verifying its bundle." diff --git a/wai/examples/apple-app/AGENTS.md b/wai/examples/apple-app/AGENTS.md new file mode 100644 index 000000000..f4067cea5 --- /dev/null +++ b/wai/examples/apple-app/AGENTS.md @@ -0,0 +1,21 @@ +# Working on this project + +This is a WAI project made from the `apple-app` template: a SwiftUI player whose stream goes +through the WAI native sink, and a recorded playback session (`sample-session.json`) replayed +through it. `Sources/WaiApp/PlayerApp.swift` is the whole integration: a `WaiSink.Config`, a +`WaiPlayerSession`, and its `player` in a `VideoPlayer`. + +- **Build.** The sink is the `wai-sink-swift` package of the WAI repository: build its macOS slice + once (`WAI_SINK_SLICES=macos scripts/build-xcframework.sh` there), set `WAI_SINK_SWIFT` to that + package's directory, then `swift build`. +- **Run.** `swift run WaiApp`, and give it an HLS address (the `package-and-deliver` template's, + under `wai dev`, by default). +- **Test.** `swift run WaiApp --replay sample-session.json` replays the recorded session through + the sink, prints how many playlists and objects went through it, and exits 0 when the bundle + verifies. Run it after every change. +- **Conform.** For a stream its owner seals, set `requireSigned: true`: nothing the owner did not + sign then reaches the decoder. Never turn that off to make a stream play. +- `wai.json` is the project file; its `$schema` names the schema editors validate it with. Keep + `name` a lowercase DNS label. +- Never put a key, a token or any other secret in this repository. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/apple-app/wai.json b/wai/examples/apple-app/wai.json index 1b6305043..7f093fd7c 100644 --- a/wai/examples/apple-app/wai.json +++ b/wai/examples/apple-app/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "apple-app", "template": "apple-app", "title": "A native app on Apple platforms", "summary": "A SwiftUI player whose stream goes through the WAI native sink, and a recorded session replayed through it." diff --git a/wai/examples/encoder-ingest/AGENTS.md b/wai/examples/encoder-ingest/AGENTS.md new file mode 100644 index 000000000..53ebdb17c --- /dev/null +++ b/wai/examples/encoder-ingest/AGENTS.md @@ -0,0 +1,20 @@ +# Working on this project + +This is a WAI project made from the `encoder-ingest` template: a program (`src/main.rs`) that +starts a relay and an ingest gateway on this machine, takes an SRT encoder's stream into a room, +and verifies each group of each track against the receipt the gateway signed as the first hop. + +- **Build.** `cargo build --release`. The first build fetches the WAI repository named in + `Cargo.toml` and compiles the relay and the gateway. +- **Run.** `cargo run --release` prints an `srt://127.0.0.1:…` address; point an SRT encoder at it + (MPEG-TS, H.264 or HEVC video and AAC audio, caller mode, the stream ID and passphrase given). +- **Test.** `cargo run --release -- --synthetic 5` sends five seconds of a synthetic program itself + and exits 0 once every group verified, non-zero otherwise. Run it after every change. +- **Conform.** A group is accepted only when its receipt verifies; never weaken that check to make + a stream pass. The gateway also takes RTMP and WHIP and serves WHEP; its configuration is + `gateway.example.toml` in the repository's `joule-moq-ingest` crate. +- `wai.json` is the project file; its `$schema` names the schema editors validate it with. Keep + `name` a lowercase DNS label. +- Never put a key, a token or any other secret in this repository. The stream's passphrase is + made at each start and printed, never stored. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/encoder-ingest/wai.json b/wai/examples/encoder-ingest/wai.json index 1341393d3..980cbb9fe 100644 --- a/wai/examples/encoder-ingest/wai.json +++ b/wai/examples/encoder-ingest/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "encoder-ingest", "template": "encoder-ingest", "title": "Stream from an encoder into a room", "summary": "An SRT encoder's stream published into a room by an ingest gateway, every group signed at the first hop and checked at the relay and the subscriber." diff --git a/wai/examples/messaging/AGENTS.md b/wai/examples/messaging/AGENTS.md new file mode 100644 index 000000000..d74225cc8 --- /dev/null +++ b/wai/examples/messaging/AGENTS.md @@ -0,0 +1,21 @@ +# Working on this project + +This is a WAI project made from the `messaging` template: an end-to-end encrypted conversation, +with threads, reactions, edits and retractions, and a call placed, rung and answered in a room. +The conversation's API is under `/msg-api/` and the call's under `/room-api/`; `index.html` holds +both pages, each taking `?service=`. + +- **Build.** Nothing to build: `wai dev` serves the page as it is, with the conversation and the + call `wai.json` asks for (`"mode": "call"`, `"messaging": true`). +- **Run.** `wai dev`, then open `http://127.0.0.1:8080/` in two browsers. +- **Test.** `wai dev --check` starts the relay, the room service, the conversation and this page on + one origin, checks each through it, prints what held and stops; it exits non-zero when one did + not. Run it after every change. +- **Conform.** Messages are records of WAI live-messaging, end-to-end encrypted: the service keeps + ciphertext, receipts and signed records only, and its history survives a restart. Never send a + message's text to the service in the clear. +- `wai.json` is the project file: `wai dev` takes its defaults from its `dev` block, and its + `$schema` names the schema editors validate it with. Keep `name` a lowercase DNS label. +- Never put a key, a token or any other secret in this repository. Development keys are made in + `.wai/` by `wai dev` and stay there; `.gitignore` keeps it out. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/messaging/wai.json b/wai/examples/messaging/wai.json index f5fa0d606..4080adef9 100644 --- a/wai/examples/messaging/wai.json +++ b/wai/examples/messaging/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "messaging", "template": "messaging", "title": "Messaging and a call", "summary": "An end-to-end encrypted conversation, and a call placed, rung and answered in a room.", diff --git a/wai/examples/package-and-deliver/AGENTS.md b/wai/examples/package-and-deliver/AGENTS.md new file mode 100644 index 000000000..b8b054560 --- /dev/null +++ b/wai/examples/package-and-deliver/AGENTS.md @@ -0,0 +1,21 @@ +# Working on this project + +This is a WAI project made from the `package-and-deliver` template: a title packed by policy +(`policy.toml`) with the reference packager, `wai-pack`, as WAI objects with a standard floor and +CMAF, HLS and DASH playlists, its package receipt verified, and the floor played in `index.html`. + +- **Build.** `sh pack.sh`: it makes the sample clip, packs it into `package/` by `policy.toml`, + signs the package receipt with `test-key.hex`, and verifies the package. +- **Run.** `wai dev`, then open `http://127.0.0.1:8080/`. +- **Test.** `wai-pack verify package` (or `wai package verify package`) checks every object, + playlist and the receipt, and exits 1 when one is refused; `wai dev --check` checks the page and + the stack. Run both after every change to the policy or the page. +- **Conform.** `policy.toml` is a packaging policy (WAI packaging §3); `wai-pack` refuses one that + breaks a rule, with the reason. Point `source` at your own clip (Y4M video, WAV audio) and put + its AV1 floor in `floor/`. +- `test-key.hex` is a published test key: a receipt signed with it proves nothing about who + packed. Sign your own titles with a key of your own, a secret kept outside this repository, and + never commit it. +- `wai.json` is the project file; its `$schema` names the schema editors validate it with. Keep + `name` a lowercase DNS label. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/package-and-deliver/wai.json b/wai/examples/package-and-deliver/wai.json index 78295b056..66dcfd076 100644 --- a/wai/examples/package-and-deliver/wai.json +++ b/wai/examples/package-and-deliver/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "package-and-deliver", "template": "package-and-deliver", "title": "Package a title and play it", "summary": "A title packed as WAI objects with a standard floor, its package receipt verified, and the floor played in a page.", diff --git a/wai/examples/remote-desktop/.gitignore b/wai/examples/remote-desktop/.gitignore index 7a3455e10..000fdd4b6 100644 --- a/wai/examples/remote-desktop/.gitignore +++ b/wai/examples/remote-desktop/.gitignore @@ -1 +1,2 @@ .wai/ +host.key diff --git a/wai/examples/remote-desktop/AGENTS.md b/wai/examples/remote-desktop/AGENTS.md new file mode 100644 index 000000000..d4d63432a --- /dev/null +++ b/wai/examples/remote-desktop/AGENTS.md @@ -0,0 +1,21 @@ +# Working on this project + +This is a WAI project made from the `remote-desktop` template: one machine's screen shared into a +room. `wai dev` runs the room in remote desktop mode; `host.sh` runs the host, `wai-remote-host`, +which captures, encodes and publishes the screen and applies the inputs it is handed, each signed. + +- **Build.** Nothing to build: `wai dev` serves the room's page, and `host.sh` runs the host from + the installed binaries. +- **Run.** `wai dev`, then in a second terminal `sh host.sh --pattern` (a test pattern; without + `--pattern`, the display itself); open `http://127.0.0.1:8080/` and press **Join**. +- **Test.** `wai dev --check` checks the room's stack through its origin and stops. A session: + `sh host.sh --pattern --seconds 20 --record session.json` writes its records, and the + independent verifier in the WAI repository checks them, + `python3 -I wai/live-conformance/remote_desktop_verify.py --room session.json`. +- **Conform.** Control passes only by a grant exchange the host proves its key for; inputs outside + a controller's classes are refused. Never hand control without it. +- `host.key` is the host's private key (made by `host.sh`, mode 0600), a secret: never commit it; + `.gitignore` keeps it out, with `.wai/`, where `wai dev` keeps its development keys. +- `wai.json` is the project file: `wai dev` takes its defaults from its `dev` block, and its + `$schema` names the schema editors validate it with. Keep `name` a lowercase DNS label. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/remote-desktop/wai.json b/wai/examples/remote-desktop/wai.json index b67d990c6..4b9297b5c 100644 --- a/wai/examples/remote-desktop/wai.json +++ b/wai/examples/remote-desktop/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "remote-desktop", "template": "remote-desktop", "title": "Remote desktop in a room", "summary": "One machine's screen shared into a room; viewers watch, a controller drives it, every input signed.", diff --git a/wai/examples/verify-receipts/AGENTS.md b/wai/examples/verify-receipts/AGENTS.md new file mode 100644 index 000000000..ee8a7cb52 --- /dev/null +++ b/wai/examples/verify-receipts/AGENTS.md @@ -0,0 +1,23 @@ +# Working on this project + +This is a WAI project made from the `verify-receipts` template: two verifiers of the receipts +WAI writes beside an object, one in Rust on the reference implementation (`rust/`) and one in plain +Python with no dependency (`python/`). Each checks the object, the group receipt that covers it and +the key that signed the group, and refuses any changed byte with its reason. + +- **Build.** `cd rust && cargo build --release`; the Python verifier needs nothing built. +- **Run.** `wai hello receipts` writes an object and its receipts in `receipts/`; then + `python3 python/verify_receipts.py receipts` and `cd rust && cargo run --release -- ../receipts`. +- **Test.** Both verifiers print `ok` and exit 0 on what `wai hello` wrote, and print `FAIL` and + exit 1 when one byte of `receipts/*.wai` or of a file in `receipts/rcpt/` is changed. Check both + after every change; `wai verify receipts` is the reference's verdict to compare with. +- **Conform.** A verifier of your own conforms against WAI's corpora with `wai conform verifier + --suite jwp --impl '' --root `; `wai conformance list` names the suites. + `wai verify` checks the signed result it writes. +- Never weaken a check to make a receipt pass; a refusal names the step and the reason. +- Never put a key, a token or any other secret in this repository; `receipts/` is what `wai hello` + wrote, and `.gitignore` keeps it out. +- `wai.json` is the project file; its `$schema` names the schema editors validate it with. Keep + `name` a lowercase DNS label. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it); its + `receipt_verify` tool is the reference's verdict. diff --git a/wai/examples/verify-receipts/wai.json b/wai/examples/verify-receipts/wai.json index dcaefef79..70ac019be 100644 --- a/wai/examples/verify-receipts/wai.json +++ b/wai/examples/verify-receipts/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "verify-receipts", "template": "verify-receipts", "title": "Verify receipts in Rust and in Python", "summary": "The receipts WAI writes beside an object, checked offline by the reference implementation and by a second one in plain Python." diff --git a/wai/examples/video-room/AGENTS.md b/wai/examples/video-room/AGENTS.md new file mode 100644 index 000000000..5e7cf3126 --- /dev/null +++ b/wai/examples/video-room/AGENTS.md @@ -0,0 +1,21 @@ +# Working on this project + +This is a WAI project made from the `video-room` template: your own page, `index.html`, with a +live room in it. The room client runs in a frame at `/room/?service=/room-api`; it makes a key in +the tab, asks the room service for a grant, joins the relay, and checks every object against its +signed receipt before it decodes it. The page's own script calls the room service's HTTP API. + +- **Build.** Nothing to build: `wai dev` serves the page as it is, beside the room client. +- **Run.** `wai dev`, then open `http://127.0.0.1:8080/` in two tabs and press **Join** in each. +- **Test.** `wai dev --check` starts the relay, the room service and this page on one origin, + checks each through it, prints what held and stops; it exits non-zero when one did not. Run it + after every change. +- **Conform.** The room's records (its descriptor, the grants, the clock's chain) are WAI live + rooms. With `wai mcp --toolset core,dev`, the `dev` tools start a development room, join it as + test principals and read its session, and `receipt_verify` checks that session as live-rooms §9 + does; quote its verdict. +- `wai.json` is the project file: `wai dev` takes its defaults from its `dev` block, and its + `$schema` names the schema editors validate it with. Keep `name` a lowercase DNS label. +- Never put a key, a token or any other secret in this repository. Development keys are made in + `.wai/` by `wai dev` and stay there; `.gitignore` keeps it out. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/video-room/wai.json b/wai/examples/video-room/wai.json index 206f70203..f52ffd267 100644 --- a/wai/examples/video-room/wai.json +++ b/wai/examples/video-room/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "video-room", "template": "video-room", "title": "A video room in one page", "summary": "Your own page with a live room in it: camera, microphone, every object checked against its receipt.", diff --git a/wai/examples/watch-together/AGENTS.md b/wai/examples/watch-together/AGENTS.md new file mode 100644 index 000000000..b8919eda0 --- /dev/null +++ b/wai/examples/watch-together/AGENTS.md @@ -0,0 +1,23 @@ +# Working on this project + +This is a WAI project made from the `watch-together` template: a room in which everyone watches +one file in sync. The room service names the file by its content hash in the room's signed policy; +each page fetches it, checks it, and keeps it at the room's position. Play, pause, seek and speed +are controls each page signs, each applied at one instant on the room's clock. + +- **Build.** Nothing to build: `wai dev` serves `index.html` and the file `wai.json` names + (`"watch"`; `tone.wav`, which `wai create` made). +- **Run.** `wai dev`, then open `http://127.0.0.1:8080/` in two tabs, press **Watch** in each and + **Play** in one. +- **Test.** `wai dev --check` starts the relay, the room service and this page on one origin, + checks each through it, prints what held and stops; it exits non-zero when one did not. Run it + after every change. +- **Conform.** A control is signed by the page that makes it and applies at the instant the room's + clock gives, for everyone (WAI co-watch): a page applies it when the room states it, not when it + is pressed. Another file: put it in the project and name it in `wai.json` (`"watch"`), with + `"watch_ms"`, its length in milliseconds, for anything but a WAV file. +- `wai.json` is the project file: `wai dev` takes its defaults from its `dev` block, and its + `$schema` names the schema editors validate it with. Keep `name` a lowercase DNS label. +- Never put a key, a token or any other secret in this repository. Development keys are made in + `.wai/` by `wai dev` and stay there; `.gitignore` keeps it out. +- WAI's tools for a coding agent: `wai mcp` (`wai setup-agent` registers it). diff --git a/wai/examples/watch-together/wai.json b/wai/examples/watch-together/wai.json index ffe1ec822..3321bf752 100644 --- a/wai/examples/watch-together/wai.json +++ b/wai/examples/watch-together/wai.json @@ -1,4 +1,7 @@ { + "$schema": "https://wai.transaction.science/schema/wai.json/1", + "wai": 1, + "name": "watch-together", "template": "watch-together", "title": "Watch together", "summary": "Everyone in the room watches one file, in sync: play, pause and seek apply for all at one instant.", diff --git a/wai/quickstart/README.md b/wai/quickstart/README.md index 04459ed6b..f3709c84b 100644 --- a/wai/quickstart/README.md +++ b/wai/quickstart/README.md @@ -9,9 +9,12 @@ of it is open source under Apache-2.0, built from the open-standards repository | `bin/wai` | The command line: `hello`, `create`, `dev`, `verify`, and the envelope tools (`inspect`, `wrap`, `unwrap`, `encode`, `decode` for the mandatory floor, `capabilities`) | | `bin/wai-relay` | The relay a room's media goes through: MOQT over QUIC, WebTransport and WebSocket | | `bin/wai-room` | The room service: a room's keys, its grants, its clock | +| `bin/wai-ingest` | The ingest gateway: WHIP, SRT and RTMP into a room, each group signed at the first hop, and WHEP out | | `bin/wai-remote-host` | A remote desktop room's host agent | | `bin/wai-pack` | The packager: a title as WAI objects, HTTP playlists and a MoQ catalog | +| `bin/wai-quantum` | The quantum command line: OpenQASM programs run on this machine, the result signed | | `share/wai/web/` | The room client a browser loads: its pages and their WebAssembly | +| `share/wai/studio/` | WAI Studio, which `wai dev` serves at `/_wai/` | ## Start @@ -26,8 +29,40 @@ wai verify ../wai-hello # the receipts hello wrote, checked `wai create --list` names every template. `wai dev --help` gives its options. Nothing here needs an account, and nothing is sent off your machine. -Keep `bin/` and `share/` side by side: `wai dev` finds the relay, the room service and the -room client beside itself. +Keep `bin/` and `share/` side by side: `wai dev` finds the relay, the room service, the room +client and Studio beside itself. Homebrew and Scoop install `wai-quantum` as a package of its own, +which `wai` depends on. + +## Ports + +What each program listens on by default, on 127.0.0.1. `wai dev` serves everything on one +origin and runs its own relay and room service on free ports behind it; the other rows are each +daemon's own default, as its example configuration or its source sets it, for running it alone. +No two of them share a port, so any of them run side by side. A deployment sets its own. + +| Port | Program | What | Set by | +|---|---|---|---| +| 8080 | `wai dev` | your page at `/`, the room service at `/room-api/`, the relay at `/moq`, Studio at `/_wai/` | `--port` | +| 4443/udp | `wai-relay` | MOQT over QUIC, and WebTransport | `quic_listen` | +| 4480 | `wai-relay` | MOQT over WebSocket | `websocket_listen` | +| 9682 | `wai-room` | the room service's HTTP API | `listen` | +| 8088 | `wai-ingest` | WHIP and WHEP | `[http] listen` | +| 1935 | `wai-ingest` | RTMP | `[rtmp] listen` | +| 1936 | `wai-ingest` | RTMPS, off until set | `[rtmp] tls_listen` | +| 5167 | `wai-ingest` | SCTE 104 automation, off until set | `[injector] listen` | +| 9000/udp | `wai-ingest` | SRT, off until set | `[contrib.srt] listen` | +| 9694 | `wai-audit` | the audit collector | `WAI_AUDIT_ADDR` | +| 8251 | `wai-ledger` | the receipt ledger | `WAI_LEDGER_ADDR` | +| 8252 | `wai-fleet` | the fleet service | `WAI_FLEET_ADDR` | +| 8241 | `wai-quantum-service` | the quantum job service | `WAI_QSVC_ADDR` | +| 9670 | `wai-world-relay` | the interactive worlds' relay | `WAI_RELAY_BIND` | +| 9690 | `wai_stitched_presentation serve` | the stitched presentation verifier | `--listen` | +| 8380 | `wai http origin` | an HTTP origin of WAI objects | `--port` | + +A program that reaches another names that one's port: the room service's `relay_url` is the +relay's WebSocket (`ws://127.0.0.1:4480/moq`), and the ingest gateway's `[relay] addr` is the +relay's QUIC endpoint (`127.0.0.1:4443`). joule-moq-room's `tests/ports.rs` holds every row to +the program that serves it. ## More diff --git a/wai/quickstart/build-studio.sh b/wai/quickstart/build-studio.sh new file mode 100755 index 000000000..b1fe77477 --- /dev/null +++ b/wai/quickstart/build-studio.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# WAI Studio's files, as `wai dev` serves them at /_wai/, built into one directory: +# +# bash wai/quickstart/build-studio.sh +# +# holds Studio's build (wai/wai-studio: node build.mjs): its interface, its worker and +# styles, index.html (the page `wai dev` serves), its core in WebAssembly under pkg/ +# (wai-studio/core through wasm-bindgen's web target), and Quantum Studio under quantum/ with +# its engine (wai-web with the feature quantum_engine) under quantum/pkg/, and BUILD.json, each +# file's SHA-256. The same files on every platform: an archive for any of them carries this +# directory as share/wai/studio/, where `wai dev` looks for it beside its own binary. +# +# Needs the wasm32-unknown-unknown target, the wasm-bindgen CLI at 0.2.127 (the version the +# crates pin), Node and npm. Run from anywhere; CARGO_TARGET_DIR is honoured, with a directory +# of its own under it for each workspace. Without it the builds go under the repository's +# target/studio/, which git ignores, as build-web.sh's go under an ignored target/: a build +# leaves the checkout as it found it. +set -euo pipefail +[ $# -eq 1 ] || { echo "usage: bash wai/quickstart/build-studio.sh " >&2; exit 2; } +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +OUT=$(mkdir -p "$1" && cd "$1" && pwd) +# Written over only when it is empty or holds an earlier build of this script. +if [ -n "$(ls -A "$OUT")" ] && [ ! -f "$OUT/BUILD.json" ]; then + echo "build-studio.sh: $OUT holds files of its own; name an empty directory" >&2 + exit 2 +fi +STUDIO="$ROOT/wai/wai-studio" +TARGET=${CARGO_TARGET_DIR:-$ROOT/target/studio} +command -v wasm-bindgen >/dev/null || { echo "wasm-bindgen CLI 0.2.127 is needed (cargo install --locked wasm-bindgen-cli --version 0.2.127)" >&2; exit 2; } +wasm-bindgen --version | grep -q ' 0\.2\.127$' || { echo "the wasm-bindgen CLI must be 0.2.127, the version the crates pin" >&2; exit 2; } +command -v npm >/dev/null || { echo "Node and npm are needed to build Studio's interface" >&2; exit 2; } +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# The floor's zstd is C, compiled for wasm32: the archive needs an archiver that indexes +# WebAssembly objects (llvm-ar), as wai-studio/e2e/run.sh finds it. +if [ -z "${AR_wasm32_unknown_unknown:-}" ]; then + for a in llvm-ar /opt/homebrew/opt/llvm/bin/llvm-ar /usr/local/opt/llvm/bin/llvm-ar; do + if command -v "$a" >/dev/null 2>&1; then export AR_wasm32_unknown_unknown="$(command -v "$a")"; break; fi + done +fi +if [ -z "${CC_wasm32_unknown_unknown:-}" ] && [ -n "${AR_wasm32_unknown_unknown:-}" ] && [ -x "$(dirname "$AR_wasm32_unknown_unknown")/clang" ]; then + export CC_wasm32_unknown_unknown="$(dirname "$AR_wasm32_unknown_unknown")/clang" +fi + +# Studio's core. +(cd "$STUDIO/core" && CARGO_TARGET_DIR="$TARGET/core" cargo build --locked --release --target wasm32-unknown-unknown) +wasm-bindgen --target web --out-dir "$WORK/core-pkg" --out-name wai_studio_core "$TARGET/core/wasm32-unknown-unknown/release/wai_studio_core.wasm" +# The quantum engine, for Quantum Studio. +(cd "$ROOT/wai/wai-web" && CARGO_TARGET_DIR="$TARGET/wai-web" cargo build --locked --release --target wasm32-unknown-unknown --features quantum_engine) +wasm-bindgen --target web --out-dir "$WORK/quantum-pkg" --out-name wai_quantum_ide "$TARGET/wai-web/wasm32-unknown-unknown/release/wai_web.wasm" +# The pages need the modules and the binaries alone, not the type declarations. +rm -f "$WORK"/core-pkg/*.d.ts "$WORK"/quantum-pkg/*.d.ts + +# Quantum Studio, then Studio, their packages from their lock files with no install script run. +(cd "$ROOT/wai/quantum-studio" && npm ci --ignore-scripts --no-audit --no-fund && node build.mjs --pkg "$WORK/quantum-pkg") +(cd "$STUDIO" && npm ci --ignore-scripts --no-audit --no-fund && node build.mjs --core-pkg "$WORK/core-pkg" --quantum "$ROOT/wai/quantum-studio/dist") + +find "$OUT" -mindepth 1 -delete +cp -R "$STUDIO/dist/." "$OUT/" +for f in index.html wai-studio.js pkg/wai_studio_core_bg.wasm quantum/pkg/wai_quantum_ide_bg.wasm BUILD.json; do + [ -f "$OUT/$f" ] || { echo "build-studio.sh: the build has no $f" >&2; exit 1; } +done +ls -l "$OUT" "$OUT/pkg" +echo "WAI Studio is in $OUT" diff --git a/wai/quickstart/build.sh b/wai/quickstart/build.sh index 67a4b7edb..a31e71a11 100755 --- a/wai/quickstart/build.sh +++ b/wai/quickstart/build.sh @@ -1,29 +1,37 @@ #!/usr/bin/env bash # The quickstart archive: the `wai` command line and what `wai dev` runs, for one target. # -# bash wai/quickstart/build.sh --web [--target ] [--version ] +# bash wai/quickstart/build.sh --web --studio [--target ] [--version ] # # Builds, each release and --locked, each from its own workspace: # wai wai/wai-rs, --no-default-features --features quickstart: the mandatory # floor, hello, create, dev and verify; no system codec library # wai-relay the relay (crate joule-moq-relay, joule-code/joule-code-rs) # wai-room the room service (crate joule-moq-room) +# wai-ingest the ingest gateway: WHIP, WHEP, SRT and RTMP into a room (crate joule-moq-ingest) # wai-remote-host a remote desktop room's host agent # wai-pack the packager -# and writes /wai--.tar.gz holding bin/ (those five), share/wai/web/ -# (the room client, from --web: bash wai/quickstart/build-web.sh makes it), README.md and -# LICENSE, with .sha256 beside it in the form `shasum -a 256 -c` reads. The version +# wai-quantum the quantum command line (wai/wai-quantum-cli), with the features its own +# release builds it with (wai-quantum-cli-release.yml): on macOS, metal, the +# state vector on the GPU, so the program install.sh puts on PATH and the one +# Homebrew installs (the wai formula depends on wai-quantum's) are one +# and writes /wai--.tar.gz holding bin/ (those seven), share/wai/web/ +# (the room client, from --web: bash wai/quickstart/build-web.sh makes it), share/wai/studio/ +# (WAI Studio, which `wai dev` serves at /_wai/, from --studio: bash +# wai/quickstart/build-studio.sh makes it), README.md and LICENSE, with .sha256 +# beside it in the form `shasum -a 256 -c` reads. The version # is wai/wai-rs's unless given; the triple is the host's unless given. Extra cargo flags for # the target (a linker, static linking) come from the environment, as cargo reads them. set -euo pipefail -usage() { echo "usage: bash wai/quickstart/build.sh --web [--target ] [--version ]" >&2; exit 2; } -[ $# -ge 3 ] || usage +usage() { echo "usage: bash wai/quickstart/build.sh --web --studio [--target ] [--version ]" >&2; exit 2; } +[ $# -ge 5 ] || usage ROOT=$(cd "$(dirname "$0")/../.." && pwd) OUT=$(mkdir -p "$1" && cd "$1" && pwd); shift -WEBDIR="" TRIPLE="" VERSION="" +WEBDIR="" STUDIODIR="" TRIPLE="" VERSION="" while [ $# -gt 0 ]; do case "$1" in --web) WEBDIR="$2"; shift 2 ;; + --studio) STUDIODIR="$2"; shift 2 ;; --target) TRIPLE="$2"; shift 2 ;; --version) VERSION="$2"; shift 2 ;; *) usage ;; @@ -31,10 +39,13 @@ while [ $# -gt 0 ]; do done [ -n "$WEBDIR" ] && [ -f "$WEBDIR/room/index.html" ] && [ -f "$WEBDIR/pkg/wai_live_web_bg.wasm" ] && [ -f "$WEBDIR/pkg/joule_moq_web_bg.wasm" ] \ || { echo "--web: a directory build-web.sh made (room/, moq/, pkg/)" >&2; exit 2; } +[ -n "$STUDIODIR" ] && [ -f "$STUDIODIR/index.html" ] && [ -f "$STUDIODIR/pkg/wai_studio_core_bg.wasm" ] && [ -f "$STUDIODIR/BUILD.json" ] \ + || { echo "--studio: a directory build-studio.sh made (index.html, pkg/, quantum/, BUILD.json)" >&2; exit 2; } HOST=$(rustc -vV | sed -n 's/^host: //p') TRIPLE=${TRIPLE:-$HOST} VERSION=${VERSION:-$(sed -n 's/^version = "\(.*\)"$/\1/p' "$ROOT/wai/wai-rs/Cargo.toml" | head -1)} EXE=""; case "$TRIPLE" in *windows*) EXE=".exe" ;; esac +QUANTUM=(); case "$TRIPLE" in *apple-darwin) QUANTUM=(--features metal) ;; esac TGT=(--target "$TRIPLE") build() { # : the binary's path, or a failure @@ -63,9 +74,12 @@ mkdir -p "$STAGE/$NAME/bin" "$STAGE/$NAME/share/wai" need "$ROOT/wai/wai-rs" wai --no-default-features --features quickstart --bin wai need "$ROOT/joule-code/joule-code-rs" wai-relay -p joule-moq-relay --bin wai-relay need "$ROOT/joule-code/joule-code-rs/crates/joule-moq-room" wai-room --bin wai-room +need "$ROOT/joule-code/joule-code-rs" wai-ingest -p joule-moq-ingest --bin wai-ingest need "$ROOT/joule-code/joule-code-rs/crates/joule-moq-remote" wai-remote-host --bin wai-remote-host need "$ROOT/wai/wai-pack" wai-pack --bin wai-pack +need "$ROOT/wai/wai-quantum-cli" wai-quantum --bin wai-quantum ${QUANTUM[@]+"${QUANTUM[@]}"} cp -R "$WEBDIR" "$STAGE/$NAME/share/wai/web" +cp -R "$STUDIODIR" "$STAGE/$NAME/share/wai/studio" cp "$ROOT/wai/quickstart/README.md" "$STAGE/$NAME/README.md" cp "$ROOT/wai/LICENSE" "$STAGE/$NAME/LICENSE" chmod 755 "$STAGE/$NAME"/bin/* @@ -74,6 +88,11 @@ chmod 755 "$STAGE/$NAME"/bin/* # binary would ship templates from another commit. if [ "$TRIPLE" = "$HOST" ]; then "$STAGE/$NAME/bin/wai-pack$EXE" fixture list > /dev/null + "$STAGE/$NAME/bin/wai-quantum$EXE" --version > /dev/null + # wai-ingest without a configuration says how it is run, and exits with a usage error. + set +e; "$STAGE/$NAME/bin/wai-ingest$EXE" > /dev/null 2> "$STAGE/ingest.err"; s=$?; set -e + [ "$s" = 2 ] && grep -q "usage: wai-ingest" "$STAGE/ingest.err" || { echo "build.sh: the packaged wai-ingest does not run" >&2; exit 1; } + rm -f "$STAGE/ingest.err" python3 - "$STAGE/$NAME/bin/wai$EXE" "$ROOT/wai/examples" <<'PY' || { echo "build.sh: the packaged wai does not hold this checkout's examples" >&2; exit 1; } import json, os, subprocess, sys, tempfile wai, ex = sys.argv[1], sys.argv[2] diff --git a/wai/quickstart/formulae.py b/wai/quickstart/formulae.py index 8aa9302d5..fbda1a4e9 100644 --- a/wai/quickstart/formulae.py +++ b/wai/quickstart/formulae.py @@ -5,8 +5,14 @@ attached to it: Writes /wai.rb (a Homebrew formula for macOS and Linux) and /wai.json (a Scoop manifest for Windows). A platform whose archive is not attached is left out, and stderr -says so; with none at all, it fails. Each archive holds bin/ and share/wai/web/, kept side by -side as `wai dev` expects them. +says so; with none at all, it fails. Each archive holds bin/ and share/wai/ (web/, the room +client, and studio/, WAI Studio), kept side by side as `wai dev` expects them. + +Both install every binary of the archive but wai-quantum, and depend on wai-quantum's own +formula and manifest (wai/wai-quantum-cli/release/formulae.py) instead: the archive carries it +for install.sh, while a package manager that installed it twice, once in each package, would +have two packages claim one command (Homebrew refuses the second link, and Scoop's two shims +overwrite each other). wai/quickstart/test/formulae_test.py holds this to build.sh. """ import json @@ -44,6 +50,11 @@ def url(archive): DESC = "WAI, an open media standard: its command line, and a local relay and room service" +# The archive's binaries the two install (build.sh packs these and wai-quantum). +BINS = ["wai", "wai-relay", "wai-room", "wai-ingest", "wai-remote-host", "wai-pack"] +# wai-quantum's own formula, in this tap, and its own manifest, in this bucket. +QUANTUM_FORMULA = "transaction-science/tap/wai-quantum" +QUANTUM_MANIFEST = "wai-quantum" HOME = "https://wai.transaction.science/docs/quickstart/" targets = { ("macos", "arm"): "aarch64-apple-darwin", @@ -71,14 +82,20 @@ formula = f'''class Wai < Formula for os_name in ("macos", "linux"): if os_name in blocks: formula += f" on_{os_name} do\n" + "".join(blocks[os_name]) + " end\n" -formula += ''' - def install - # bin/ and share/wai/web/ side by side: `wai dev` finds the relay, the room service and the - # room client beside its own binary. - bin.install Dir["bin/*"] - (share/"wai").install "share/wai/web" - end +installs = ", ".join(f'"bin/{b}"' for b in BINS) +formula += f''' + # The quantum command line is a formula of its own: one copy, whichever is installed first. + depends_on "{QUANTUM_FORMULA}" + def install + # bin/ and share/wai/ side by side: `wai dev` finds the relay, the room service, the room + # client and WAI Studio beside its own binary. The archive's bin/wai-quantum is install.sh's; + # here it is the dependency's. + bin.install {installs} + (share/"wai").install "share/wai/web", "share/wai/studio" + end +''' +formula += ''' test do assert_match "video-room", shell_output("#{bin}/wai create --list") assert_match "delivered-verified", shell_output("#{bin}/wai hello #{testpath}/hello") @@ -95,7 +112,9 @@ if win_sha: "homepage": HOME, "license": "Apache-2.0", "architecture": {"64bit": {"url": url(win + ".zip"), "hash": win_sha, "extract_dir": win}}, - "bin": ["bin\\wai.exe", "bin\\wai-relay.exe", "bin\\wai-room.exe", "bin\\wai-remote-host.exe", "bin\\wai-pack.exe"], + # The zip's bin\\wai-quantum.exe is left to the dependency's own manifest and shim. + "depends": QUANTUM_MANIFEST, + "bin": [f"bin\\{b}.exe" for b in BINS], } pathlib.Path(out, "wai.json").write_text(json.dumps(manifest, indent=2) + "\n") print(f"wrote the formula{' and the manifest' if win_sha else ''} for wai {version}") diff --git a/wai/quickstart/install.sh b/wai/quickstart/install.sh index c4f95c253..087377045 100755 --- a/wai/quickstart/install.sh +++ b/wai/quickstart/install.sh @@ -5,7 +5,7 @@ # # It picks the archive for this system (macOS or Linux, arm64 or x86_64), checks it against the # SHA-256 the release states, and unpacks it into $WAI_HOME (default ~/.wai): bin/ and -# share/wai/web/ side by side. Then it says which line puts ~/.wai/bin on your PATH. Nothing +# share/wai/ (the room client and WAI Studio) side by side. Then it says which line puts ~/.wai/bin on your PATH. Nothing # else is changed. WAI_VERSION picks a release (default: the one below). set -eu VERSION="${WAI_VERSION:-0.4.0}" diff --git a/wai/quickstart/test/clean-machine.sh b/wai/quickstart/test/clean-machine.sh index c1ff7be6e..5fd3cddaa 100755 --- a/wai/quickstart/test/clean-machine.sh +++ b/wai/quickstart/test/clean-machine.sh @@ -6,15 +6,18 @@ # A machine with nothing of WAI on it: a fresh HOME and working directory, a PATH of the system's # own directories, no cargo and no repository. The steps are the quickstart's: # 1. check the archive against its .sha256 beside it, and unpack it; -# 2. wai hello; 3. wai verify; 4. wai create my-room --template video-room; -# 5. wai dev --check in the project: the relay, the room service and the page on one origin, -# each checked through it; -# 6. with --browser (BROWSER_BIN and PLAYWRIGHT_HOST set): the project's page joined in two +# 2. the archive holds what `wai dev` and the quickstart name: bin/ with wai, wai-relay, +# wai-room, wai-ingest, wai-remote-host, wai-pack and wai-quantum, each of which runs, and +# share/wai/web/ and share/wai/studio/ (Studio's page and its core); +# 3. wai hello; 4. wai verify; 5. wai create my-room --template video-room; +# 6. wai dev --check in the project: the relay, the room service, the page and WAI Studio at +# /_wai/ on one origin, each checked through it; +# 7. with --browser (BROWSER_BIN and PLAYWRIGHT_HOST set): the project's page joined in two # browser pages, each decoding the other (the video-room example's test). # It prints each step's time, the total, and a JSON line; it exits non-zero when a step fails, # naming it. Downloading the archive is not timed: the archive is given. set -uo pipefail -[ $# -ge 1 ] || { sed -n '2,16p' "$0"; exit 2; } +[ $# -ge 1 ] || { sed -n '2,19p' "$0"; exit 2; } ARCHIVE=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") BROWSER=0; [ "${2:-}" = --browser ] && BROWSER=1 HERE=$(cd "$(dirname "$0")" && pwd) @@ -47,10 +50,26 @@ unpack() { step unpack unpack BIN=$(ls -d "$ROOT"/work/wai-*/bin 2>/dev/null | head -1) export PATH="$BIN:$PATH" +contents() { + local b missing="" + for b in wai wai-relay wai-room wai-ingest wai-remote-host wai-pack wai-quantum; do + [ -x "$BIN/$b" ] || missing="$missing bin/$b" + done + for f in share/wai/web/room/index.html share/wai/studio/index.html share/wai/studio/pkg/wai_studio_core_bg.wasm; do + [ -f "$BIN/../$f" ] || missing="$missing $f" + done + [ -z "$missing" ] || { echo "the archive lacks:$missing"; return 1; } + wai-quantum --version || return 1 + # wai-ingest without a configuration says how it is run, and exits 2. + local s=0; wai-ingest 2> "$ROOT/ingest.err" || s=$? + [ "$s" = 2 ] && grep -q "usage: wai-ingest" "$ROOT/ingest.err" +} +[ -z "$failed" ] && step contents contents [ -z "$failed" ] && step hello wai hello [ -z "$failed" ] && step verify wai verify wai-hello [ -z "$failed" ] && step create wai create my-room --template video-room -[ -z "$failed" ] && step dev-check bash -c 'cd my-room && wai dev --check --port 0' +# Every piece held, Studio among them: the archive's share/wai/studio, found beside wai. +[ -z "$failed" ] && step dev-check bash -c 'cd my-room && { wai dev --check --port 0 > ../dev-check.out; s=$?; cat ../dev-check.out; [ "$s" = 0 ] && grep -q "ok WAI Studio at /_wai/" ../dev-check.out; }' if [ -z "$failed" ] && [ "$BROWSER" = 1 ]; then step two-tabs env WAI_BIN="$BIN/wai" node "$HERE/../../examples/video-room/test/video-room.test.mjs" fi diff --git a/wai/quickstart/test/formulae_test.py b/wai/quickstart/test/formulae_test.py new file mode 100644 index 000000000..5864d30d5 --- /dev/null +++ b/wai/quickstart/test/formulae_test.py @@ -0,0 +1,140 @@ +"""The package managers' entries for a release of `wai`, as wai/quickstart/formulae.py writes +them, held to the archive build.sh packs: + + python3 -I wai/quickstart/test/formulae_test.py + +It runs formulae.py against a stand-in for the forge's release API on 127.0.0.1 (a release +with every platform's archive and its .sha256 attached), and checks what it writes: + +- the Homebrew formula installs every binary build.sh packs but wai-quantum, and depends on + wai-quantum's own formula, which wai/wai-quantum-cli/release/formulae.py writes into the same + tap: installed by both, the one command would belong to two formulae, and the second's link + would fail; +- the Scoop manifest names the same binaries, each in the zip the Windows job packs, and + depends on wai-quantum's own manifest, so the two never share a shim; +- the archive's wai-quantum is built with the features wai-quantum's own release builds it + with on each platform (metal on macOS), so the program install.sh installs is the one the + package managers do; +- with ruby on PATH, the formula parses as Ruby. + +Exit 0 when each holds, 1 naming each that does not. Standard library only. +""" + +import hashlib +import http.server +import json +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import threading + +ROOT = pathlib.Path(__file__).resolve().parents[3] +QUICKSTART = ROOT / "wai/quickstart" +failures = [] + + +def check(ok, what): + print(("ok " if ok else "FAIL ") + what) + if not ok: + failures.append(what) + + +build_sh = (QUICKSTART / "build.sh").read_text() +# `need "" `: what the archive's bin/ holds. +packed = re.findall(r'^need "[^"]+" (\S+)', build_sh, re.M) +check("wai" in packed and "wai-quantum" in packed and len(packed) == len(set(packed)), f"build.sh packs wai and wai-quantum, each once ({packed})") +installed = [b for b in packed if b != "wai-quantum"] + +# The forge, as formulae.py reads it: the release by its tag, and each archive's .sha256. +VERSION, TAG, REPO = "9.9.9", "wai-v9.9.9", "Transaction-Science/open-standards" +TRIPLES = ["aarch64-apple-darwin", "x86_64-apple-darwin", "aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl"] +archives = [f"wai-{VERSION}-{t}.tar.gz" for t in TRIPLES] + [f"wai-{VERSION}-x86_64-pc-windows-msvc.zip"] +files = {} +for a in archives: + files[a] = a.encode() + files[a + ".sha256"] = f"{hashlib.sha256(a.encode()).hexdigest()} {a}\n".encode() + + +class Forge(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == f"/api/v1/repos/{REPO}/releases/tags/{TAG}": + body = json.dumps({"tag_name": TAG, "assets": [{"name": n} for n in files]}).encode() + elif self.path.startswith(f"/{REPO}/releases/download/{TAG}/") and self.path.rsplit("/", 1)[1] in files: + body = files[self.path.rsplit("/", 1)[1]] + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Forge) +threading.Thread(target=server.serve_forever, daemon=True).start() +api = f"http://127.0.0.1:{server.server_address[1]}/api/v1" +with tempfile.TemporaryDirectory() as out: + run = subprocess.run( + [sys.executable, "-I", str(QUICKSTART / "formulae.py"), VERSION, TAG, api, REPO, "https://forge.example/" + REPO, out], + capture_output=True, text=True, timeout=120, + ) + server.shutdown() + check(run.returncode == 0, f"formulae.py writes the entries ({run.stderr.strip() or run.stdout.strip()})") + rb = pathlib.Path(out, "wai.rb").read_text() if run.returncode == 0 else "" + manifest = json.loads(pathlib.Path(out, "wai.json").read_text()) if run.returncode == 0 else {} + + # The formula. + lines = [l.strip() for l in rb.splitlines() if l.strip().startswith("bin.install")] + check(len(lines) == 1, f"the formula installs bin/ in one line ({lines})") + names = re.findall(r'"bin/([^"]+)"', lines[0]) if lines else [] + check(names == installed, f"the formula installs each binary build.sh packs but wai-quantum: {names}, not {installed}") + check('Dir["bin/*"]' not in rb and '"bin/wai-quantum"' not in rb, "the formula does not install bin/wai-quantum, by name or by a glob") + check('depends_on "transaction-science/tap/wai-quantum"' in rb, "the formula depends on wai-quantum's own formula") + for t in TRIPLES: + check(f"wai-{VERSION}-{t}.tar.gz" in rb and hashlib.sha256(f"wai-{VERSION}-{t}.tar.gz".encode()).hexdigest() in rb, f"the formula names the {t} archive and its digest") + if shutil.which("ruby"): + with tempfile.NamedTemporaryFile("w", suffix=".rb") as f: + f.write(rb) + f.flush() + r = subprocess.run(["ruby", "-c", f.name], capture_output=True, text=True) + check(r.returncode == 0, f"the formula parses as Ruby ({r.stdout.strip() or r.stderr.strip()})") + else: + print("note no ruby on PATH; the formula's syntax is not checked") + + # The manifest. + check(manifest.get("bin") == [f"bin\\{b}.exe" for b in installed], f"the manifest names each binary but wai-quantum: {manifest.get('bin')}") + check(manifest.get("depends") == "wai-quantum", f"the manifest depends on wai-quantum's own manifest ({manifest.get('depends')!r})") + +# The Windows job packs the binaries build.sh does, so each the manifest names is in the zip. +release = (ROOT / ".github/workflows/wai-release.yml").read_text() +windows = re.findall(r"bin = '([^']+)'", release) +check(sorted(windows) == sorted(packed), f"the Windows job packs build.sh's binaries ({windows})") + +# wai-quantum's own entries are where the dependency names them: in this tap and this bucket, +# as wai-quantum. +own = (ROOT / "wai/wai-quantum-cli/release/formulae.py").read_text() +check("class WaiQuantum < Formula" in own and 'pathlib.Path(out, "wai-quantum.rb")' in own and 'pathlib.Path(out, "wai-quantum.json")' in own, + "wai-quantum's release writes the formula wai-quantum and the manifest wai-quantum") + +# One program by either path: the archive's wai-quantum has the features of wai-quantum's own +# release, on macOS (its macOS job) and elsewhere (none). +quantum = (ROOT / ".github/workflows/wai-quantum-cli-release.yml").read_text() +mac_job = quantum.split("\n macos:", 1)[1].split("\n linux:", 1)[0] +mac_features = re.findall(r"cargo build [^\n]*--features (\S+)", mac_job) +other = quantum.split("\n linux:", 1)[1] +check(re.findall(r"cargo build [^\n]*--features (\S+)", other) == [], "wai-quantum's own Linux and Windows builds take no feature") +m = re.search(r"\*apple-darwin\) QUANTUM=\(--features (\S+)\)", build_sh) +check(m is not None and mac_features == [m.group(1)], f"the archive's macOS wai-quantum has the features of its own release's: {m and m.group(1)}, {mac_features}") +check(re.search(r'^need "\$ROOT/wai/wai-quantum-cli" wai-quantum --bin wai-quantum \$\{QUANTUM\[@\]', build_sh, re.M) is not None, + "build.sh builds wai-quantum with those features") + +if failures: + print(f"formulae_test: {len(failures)} check(s) failed") + sys.exit(1) +print("formulae_test: every check held") diff --git a/wai/registry/capabilities.json b/wai/registry/capabilities.json index 18c3d268a..f2e3c3b8a 100644 --- a/wai/registry/capabilities.json +++ b/wai/registry/capabilities.json @@ -2954,6 +2954,13 @@ "section": "§4.1", "status": "defined" }, + { + "name": "WHD1", + "kind": "magic", + "document": "extensions/holdings.md", + "section": "§4.1", + "status": "defined" + }, { "name": "WHP1", "kind": "magic", diff --git a/wai/schema/README.md b/wai/schema/README.md new file mode 100644 index 000000000..8da735959 --- /dev/null +++ b/wai/schema/README.md @@ -0,0 +1,29 @@ +# Project file schemas + +[`wai-json-1.schema.json`](wai-json-1.schema.json) is the JSON Schema of `wai.json`, a WAI +project file, version 1 (JSON Schema 2020-12). Its `$id`, and the `$schema` member every project +file `wai create` writes names, is `https://wai.transaction.science/schema/wai.json/1`, the +address it is to be served from. The release of `wai` waits for it: the tag build +(`.github/workflows/wai-release.yml`) fails while that address does not serve this file, so no +release writes a project file that names a schema an editor or an agent cannot fetch. + +`wai/wai-rs/src/bin/wai_cli/dx/project.rs` is the reader. Its tests hold this file to it: the +same members in the file and in its `dev` block, the same modes, the same name rule, and the +same verdict, accept or refuse, on every template's project file and on a set of files each +breaking one rule. A change to either is a change to both. + +## Versions + +Version 1 stays open until the first release of `wai` that writes it. From that release on, its +readers are in use, and a reader refuses any member it does not know, so version 1 does not +change: a member added to it would be refused, as unknown, by every reader of that release. A +later version is a new schema at `…/schema/wai.json/`, beside this one. A reader +refuses a file of a later version as that version (`upgrade wai`), by its `wai` member or by its +`$schema`, before it reads any other member, so the members the later version adds never stop +it first. + +The project file's other members in the toolchain's design (`requires`, `room`, `package`, +`conform`, `breaking`, `targets`, `receipts`, and `port` in `dev`) are version 2's. Each joins +the file with the operation that reads it, as every member of version 1 has its reader; a member +accepted with nothing to read it would be one the strict reader ignores, which is what it +refuses to do. diff --git a/wai/schema/wai-json-1.schema.json b/wai/schema/wai-json-1.schema.json new file mode 100644 index 000000000..870894c9c --- /dev/null +++ b/wai/schema/wai-json-1.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://wai.transaction.science/schema/wai.json/1", + "title": "wai.json, a WAI project file, version 1", + "description": "The project file `wai create` writes and `wai dev`, `wai app check` and `wai create` read. Every member is optional; a member not listed here is refused. Paths are relative to the file's directory. A whole number may be written with a zero fraction (4.0 is 4), and none is above 9007199254740991 (2^53 - 1), the largest every JSON reader holds exactly. Secrets never belong in it.", + "type": "object", + "additionalProperties": false, + "properties": { + "$schema": { + "description": "This schema: an editor or an agent validates the file with it.", + "const": "https://wai.transaction.science/schema/wai.json/1" + }, + "wai": { + "description": "The project file's version. A file without it is of version 1; a reader of version 1 refuses a later one.", + "const": 1 + }, + "name": { + "description": "The project's machine name, the name it is deployed by: a DNS label, lowercase letters, digits and hyphens, at most 63, beginning with a letter and not ending with a hyphen. `wai create` names a project for its directory.", + "type": "string", + "pattern": "^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$" + }, + "template": { + "description": "The template the project was made from.", + "type": "string" + }, + "title": { + "description": "The project's title, for people.", + "type": "string" + }, + "summary": { + "description": "One line on what the project does.", + "type": "string" + }, + "app": { + "description": "A program made with Create (app.oui), which the project's page renders: a path relative to the file.", + "type": "string" + }, + "dev": { + "description": "What `wai dev` runs in the project's directory; a flag on its command line wins over each member.", + "type": "object", + "additionalProperties": false, + "properties": { + "site": { + "description": "The directory served at /, the project's own page: a path relative to the file.", + "type": "string" + }, + "page": { + "description": "The page `wai dev` names first when it is ready, a path on its origin.", + "type": "string" + }, + "mode": { + "description": "The room `wai dev` runs: a room, a watch room, a call, a remote desktop room, or a show (a room with co-performance: hosts on camera and an audience that votes).", + "enum": ["room", "watch", "call", "remote", "show"] + }, + "publishers": { + "description": "Publishers at once.", + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + }, + "watch": { + "description": "A watch room's source: a path relative to the file.", + "type": "string" + }, + "watch_ms": { + "description": "The source's duration in milliseconds, where it is not a WAV file.", + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "messaging": { + "description": "A conversation beside the room.", + "type": "boolean" + }, + "e2ee": { + "description": "The room end-to-end encrypted.", + "type": "boolean" + }, + "audience": { + "description": "A show's inputs per viewer per poll; a show's alone.", + "type": "integer", + "minimum": 1, + "maximum": 255 + } + }, + "if": { "required": ["audience"] }, + "then": { "required": ["mode"], "properties": { "mode": { "const": "show" } } } + } + } +} diff --git a/wai/wai-rs/Cargo.lock b/wai/wai-rs/Cargo.lock index 5e0a78394..ca778ae8a 100644 --- a/wai/wai-rs/Cargo.lock +++ b/wai/wai-rs/Cargo.lock @@ -4725,6 +4725,7 @@ dependencies = [ "pkg-config", "raptorq", "rav1e", + "regex", "rustls", "rustls-platform-verifier", "serde", diff --git a/wai/wai-rs/Cargo.toml b/wai/wai-rs/Cargo.toml index 8cc3bcd14..d93f09b10 100644 --- a/wai/wai-rs/Cargo.toml +++ b/wai/wai-rs/Cargo.toml @@ -1869,6 +1869,9 @@ wai-ops = { path = "../wai-ops", features = ["oracle"] } # RaptorQ object of the carousel corpus with and decodes from what a receiver within its loss # bound holds. raptorq = "2.0.1" +# The project file's schema (wai/schema/) states its name rule as a regular expression; the +# reader's tests run it, as a validator of the schema does, beside the reader's own rule. +regex = "1" [build-dependencies] pkg-config = "0.3" # locate system libflac (cross-platform; replaces diff --git a/wai/wai-rs/src/bin/wai_cli/dx/create.rs b/wai/wai-rs/src/bin/wai_cli/dx/create.rs index 94f9037a5..8ab15f59c 100644 --- a/wai/wai-rs/src/bin/wai_cli/dx/create.rs +++ b/wai/wai-rs/src/bin/wai_cli/dx/create.rs @@ -97,13 +97,17 @@ fn names() -> String { templates::TEMPLATES.iter().map(|t| t.name).collect::>().join(", ") } -/// The template's files in `dir`, which must not exist or must be empty. A project file is -/// written where the template carries none. +/// The template's files in `dir`, which must not exist or must be empty. The project is named +/// for its directory when that is a project name, else for the template ([`project::name_for`]): +/// its project file is the template's with that `name`, or one written where the template +/// carries none. fn write(t: &Template, dir: &Path) -> Result<(), String> { + let name = project::name_for(dir, t.name); let mut files: Vec<(String, Vec)> = t.files.iter().map(|(p, b)| (p.to_string(), b.to_vec())).collect(); files.extend(t.generated.iter().map(|(p, f)| (p.to_string(), f()))); - if !files.iter().any(|(p, _)| p == project::FILE) { - files.push((project::FILE.into(), project::minimal(t.name).into_bytes())); + match files.iter_mut().find(|(p, _)| p == project::FILE) { + Some((_, b)) => *b = project::renamed(&String::from_utf8_lossy(b), &name).map_err(|e| format!("{}'s {}: {e}", t.name, project::FILE))?.into_bytes(), + None => files.push((project::FILE.into(), project::minimal(&name, t.name).into_bytes())), } write_project(dir, files) } @@ -178,7 +182,8 @@ pub(super) fn export_app(program: &str, dir: &Path, runtime: Option<&Path>) -> R if let Some(a) = p.audience { dev.push(("audience", a.into())); } - let top: Vec<(&str, serde_json::Value)> = vec![("title", p.title.clone().into()), ("summary", p.summary.clone().into()), ("app", wai_app::export::APP.into())]; + let mut top = project::head(&project::name_for(dir, "app")); + top.extend([("title", p.title.clone().into()), ("summary", p.summary.clone().into()), ("app", wai_app::export::APP.into())]); files.push((project::FILE.into(), project::render(&top, &dev).into_bytes())); let names: Vec = files.iter().map(|(p, _)| p.clone()).chain([LICENSE.0.to_string()]).collect(); write_project(dir, files)?; @@ -240,6 +245,8 @@ mod tests { write(t, &d).unwrap_or_else(|e| panic!("{}: {e}", t.name)); let p = project::load(&d).unwrap().unwrap(); assert_eq!(p.template.as_deref(), Some(t.name), "{}: its project file names it", t.name); + // Named for its directory, which is a project name here. + assert_eq!(p.name, Some(d.file_name().unwrap().to_str().unwrap().to_string()), "{}", t.name); assert!(d.join("README.md").is_file(), "{}: a README", t.name); // Every project states its licence: the file, and a line of its README. assert_eq!(std::fs::read(d.join("LICENSE")).unwrap(), LICENSE.1, "{}: LICENSE", t.name); @@ -249,6 +256,22 @@ mod tests { } } + /// Every template carries the instructions a coding agent reads, AGENTS.md: what the project + /// is, and how to build it, run it, test it and conform it, and WAI's tools for an agent. + #[test] + fn every_template_carries_an_agents_md_with_its_commands() { + for t in templates::TEMPLATES { + let (_, a) = t.files.iter().find(|(p, _)| *p == "AGENTS.md").unwrap_or_else(|| panic!("{}: no AGENTS.md", t.name)); + let a = std::str::from_utf8(a).unwrap(); + assert!(a.starts_with("# Working on this project\n"), "{}", t.name); + assert!(a.contains(&format!("the `{}` template", t.name)), "{}: AGENTS.md names its template", t.name); + for section in ["**Build.**", "**Run.**", "**Test.**", "**Conform.**"] { + assert!(a.contains(section), "{}: AGENTS.md says {section}", t.name); + } + assert!(a.contains("`wai mcp`") && a.contains("secret"), "{}: WAI's tools, and no secret in the repository", t.name); + } + } + #[test] fn a_directory_that_holds_anything_is_never_written_to() { let d = tmp("full"); diff --git a/wai/wai-rs/src/bin/wai_cli/dx/project.rs b/wai/wai-rs/src/bin/wai_cli/dx/project.rs index 742c4d62d..ed3034cd8 100644 --- a/wai/wai-rs/src/bin/wai_cli/dx/project.rs +++ b/wai/wai-rs/src/bin/wai_cli/dx/project.rs @@ -3,7 +3,10 @@ //! module is the one place it is named, read and written ([`render`]). //! //! ```json -//! { "template": "video-room", +//! { "$schema": "https://wai.transaction.science/schema/wai.json/1", +//! "wai": 1, +//! "name": "my-room", +//! "template": "video-room", //! "title": "A video room in one page", //! "app": "app.oui", //! "dev": { "site": ".", "page": "/", "mode": "room", "publishers": 4, @@ -11,15 +14,55 @@ //! "audience": 2 } } //! ``` //! -//! Every member is optional. `dev` gives `wai dev` its defaults; a flag on the command line -//! wins over each. Paths are relative to the file's directory. A member this reader does -//! not know is refused, so a misspelt one is never silently ignored. +//! `$schema` names the file's JSON Schema ([`SCHEMA`]; the file is +//! `wai/schema/wai-json-1.schema.json`), which an editor or an agent validates the file with. +//! `wai` is the file's version ([`VERSION`]). A file of a later version, by its `wai` or by its +//! `$schema`, is refused as that version (`upgrade wai`) before any other member is read: a +//! later version adds members this reader does not know, and the file is never half read or +//! refused member by member. `name` is the project's machine name, the name it is deployed by: +//! a DNS label ([`valid_name`]). +//! +//! Every member is optional; a file without `wai` is of version 1. `dev` gives `wai dev` its +//! defaults; a flag on the command line wins over each. Paths are relative to the file's +//! directory. A member this reader does not know is refused, so a misspelt one is never +//! silently ignored. use serde_json::{Map, Value}; use std::path::{Path, PathBuf}; /// The project file's name. pub const FILE: &str = "wai.json"; +/// The project file's version this reader reads, its `wai` member. +pub const VERSION: u64 = 1; +/// The URL of the JSON Schema of [`VERSION`], its `$schema` member and the schema's `$id`: +/// [`SCHEMA_OF`] and the version. +pub const SCHEMA: &str = "https://wai.transaction.science/schema/wai.json/1"; +/// What the URL of the JSON Schema of each version begins with; the version follows it. +pub const SCHEMA_OF: &str = "https://wai.transaction.science/schema/wai.json/"; +/// The members a project file of [`VERSION`] has, in the order [`render`] writes them, and those +/// of its `dev` block. +pub const MEMBERS: &[&str] = &["$schema", "wai", "name", "template", "title", "summary", "app", "dev"]; +pub const DEV_MEMBERS: &[&str] = &["site", "page", "mode", "publishers", "watch", "watch_ms", "messaging", "e2ee", "audience"]; +/// What [`valid_name`] accepts, as the schema states it. +pub const NAME_PATTERN: &str = "^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$"; + +/// A project's name: a DNS label, so that it can name what the project is deployed as. Lowercase +/// ASCII letters, digits and hyphens, at most 63 of them, beginning with a letter and ending with +/// a letter or a digit ([`NAME_PATTERN`]). +pub fn valid_name(n: &str) -> bool { + let b = n.as_bytes(); + !b.is_empty() + && b.len() <= 63 + && b[0].is_ascii_lowercase() + && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') + && b[b.len() - 1] != b'-' +} + +/// The name a project made in `dir` takes: the directory's own name when it is a valid one +/// ([`valid_name`]), else `fallback`. +pub fn name_for(dir: &Path, fallback: &str) -> String { + dir.file_name().and_then(|n| n.to_str()).filter(|n| valid_name(n)).unwrap_or(fallback).to_string() +} /// What `wai dev` takes from a project file. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -48,6 +91,8 @@ pub struct Dev { pub struct Project { /// The directory the file is in; the file's relative paths resolve against it. pub root: PathBuf, + /// The project's machine name ([`valid_name`]). + pub name: Option, pub template: Option, pub title: Option, pub summary: Option, @@ -71,7 +116,25 @@ pub fn load(dir: &Path) -> Result, String> { pub fn parse(root: &Path, text: &str) -> Result { let v: Value = serde_json::from_str(text).map_err(|e| format!("not JSON: {e}"))?; let top = v.as_object().ok_or("not a JSON object")?; - only(top, &["template", "title", "summary", "app", "dev"], "")?; + // The version first, by `wai` and then by `$schema`: a file of a later version has members + // this reader does not know, and is refused as that version, not by its first such member. + match top.get("wai") { + None => {} + Some(v) => match whole(v) { + Some(VERSION) => {} + Some(n) if n > VERSION => { + return Err(format!("wai: a project file of version {n}; this wai reads version {VERSION}: upgrade wai")); + } + _ => return Err(format!("wai: the project file's version, {VERSION}")), + }, + } + if let Some(u) = top.get("$schema").and_then(Value::as_str) + && let Some(n) = u.strip_prefix(SCHEMA_OF).and_then(|n| n.parse::().ok()) + && n > VERSION + { + return Err(format!("$schema: {u}, the schema of a project file of version {n}; this wai reads version {VERSION}: upgrade wai")); + } + only(top, MEMBERS, "")?; let s = |m: &Map, k: &str| -> Result, String> { match m.get(k) { None => Ok(None), @@ -79,8 +142,22 @@ pub fn parse(root: &Path, text: &str) -> Result { Some(_) => Err(format!("{k}: a string")), } }; + if let Some(u) = s(top, "$schema")? + && u != SCHEMA + { + return Err(format!("$schema: {SCHEMA}, the schema of the version this wai reads, not {u}")); + } + let name = s(top, "name")?; + if let Some(n) = &name + && !valid_name(n) + { + return Err(format!( + "name: {n:?} is not a project name: lowercase letters, digits and hyphens, at most 63, beginning with a letter and not ending with a hyphen" + )); + } let mut p = Project { root: root.to_path_buf(), + name, template: s(top, "template")?, title: s(top, "title")?, summary: s(top, "summary")?, @@ -89,12 +166,12 @@ pub fn parse(root: &Path, text: &str) -> Result { }; if let Some(d) = top.get("dev") { let d = d.as_object().ok_or("dev: an object")?; - only(d, &["site", "page", "mode", "publishers", "watch", "watch_ms", "messaging", "e2ee", "audience"], "dev.")?; + only(d, DEV_MEMBERS, "dev.")?; let path = |k: &str| -> Result, String> { Ok(s(d, k).map_err(|e| format!("dev.{e}"))?.map(|x| root.join(x))) }; let num = |k: &str| -> Result, String> { match d.get(k) { None => Ok(None), - Some(n) => n.as_u64().map(Some).ok_or(format!("dev.{k}: a whole number")), + Some(n) => whole(n).map(Some).ok_or(format!("dev.{k}: a whole number, at most {WHOLE_MAX}")), } }; let boolean = |k: &str| -> Result, String> { @@ -129,10 +206,37 @@ pub fn parse(root: &Path, text: &str) -> Result { /// The modes `wai dev` runs. pub const MODES: &[&str] = &["room", "watch", "call", "remote", "show"]; -/// The text of a project file naming `template`, for `wai create` to write when the -/// template carries none of its own. -pub fn minimal(template: &str) -> String { - render(&[("template", Value::from(template))], &[]) +/// The members every project file `wai create` writes begins with: its schema, its version and +/// the project's `name`. +pub fn head(name: &str) -> Vec<(&'static str, Value)> { + vec![("$schema", Value::from(SCHEMA)), ("wai", Value::from(VERSION)), ("name", Value::from(name))] +} + +/// The text of a project file of the project `name`, made from `template`, for `wai create` to +/// write when the template carries none of its own. +pub fn minimal(name: &str, template: &str) -> String { + let mut top = head(name); + top.push(("template", Value::from(template))); + render(&top, &[]) +} + +/// A template's project file, `text`, as the project `name` made from it has it: its `name` +/// member that name, and everything else as the template has it, laid out by [`render`]. The +/// file is read first, so a template whose file this reader refuses is refused here. +pub fn renamed(text: &str, name: &str) -> Result { + parse(Path::new("."), text)?; + let v: Value = serde_json::from_str(text).map_err(|e| format!("not JSON: {e}"))?; + let top = v.as_object().ok_or("not a JSON object")?; + let mut members: Vec<(&str, Value)> = top.iter().filter(|(k, _)| *k != "dev").map(|(k, x)| (k.as_str(), x.clone())).collect(); + match members.iter_mut().find(|(k, _)| *k == "name") { + Some(m) => m.1 = Value::from(name), + None => { + let at = members.iter().position(|(k, _)| !["$schema", "wai"].contains(k)).unwrap_or(members.len()); + members.insert(at, ("name", Value::from(name))); + } + } + let dev: Vec<(&str, Value)> = top.get("dev").and_then(Value::as_object).map(|d| d.iter().map(|(k, x)| (k.as_str(), x.clone())).collect()).unwrap_or_default(); + Ok(render(&members, &dev)) } /// A project file's text, in the layout every template's own has: its members in the order @@ -147,6 +251,28 @@ pub fn render(top: &[(&str, Value)], dev: &[(&str, Value)]) -> String { format!("{{\n{}\n}}\n", lines.join(",\n")) } +/// The largest whole number the file holds: 2^53 - 1, the largest that every JSON reader holds +/// exactly (a reader that keeps its numbers in double precision, as a browser does, holds no +/// larger one), and the schema's `maximum` where it states none smaller. +pub const WHOLE_MAX: u64 = (1 << 53) - 1; + +/// A whole number, 0 to [`WHOLE_MAX`], however it is written: `4`, `4.0` and `4e0` are one +/// number, as JSON Schema's `integer` and `const` take them, so a file the schema accepts this +/// reader reads. +fn whole(v: &Value) -> Option { + let n = match v.as_u64() { + Some(n) => n, + None => { + let f = v.as_f64()?; + if f.fract() != 0.0 || !(0.0..=WHOLE_MAX as f64).contains(&f) { + return None; + } + f as u64 + } + }; + (n <= WHOLE_MAX).then_some(n) +} + fn only(m: &Map, known: &[&str], at: &str) -> Result<(), String> { match m.keys().find(|k| !known.contains(&k.as_str())) { Some(k) => Err(format!("{at}{k}: not a member this version reads ({})", known.join(", "))), @@ -178,23 +304,247 @@ mod tests { #[test] fn the_minimal_file_reads_back() { - assert_eq!(parse(Path::new("/p"), &minimal("video-room")).unwrap().template.as_deref(), Some("video-room")); + let p = parse(Path::new("/p"), &minimal("my-room", "video-room")).unwrap(); + assert_eq!((p.name.as_deref(), p.template.as_deref()), (Some("my-room"), Some("video-room"))); } - /// The writer lays a file out as the templates' own files are laid out, byte for byte. + fn examples() -> Vec<(String, String)> { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples"); + let mut v: Vec<(String, String)> = std::fs::read_dir(&dir) + .unwrap() + .flatten() + .filter(|e| e.path().is_dir()) + .map(|e| (e.file_name().to_string_lossy().into_owned(), std::fs::read_to_string(e.path().join(FILE)).unwrap())) + .collect(); + v.sort(); + assert!(!v.is_empty()); + v + } + + /// The writer lays a file out as the templates' own files are laid out, byte for byte, and + /// a project made from a template under the template's own name has its file unchanged. #[test] fn the_writer_writes_a_templates_file_as_it_is() { - let ex = |t: &str| std::fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples").join(t).join(FILE)).unwrap(); - for t in ["watch-together", "messaging", "android-app"] { - let p = parse(Path::new("/p"), &ex(t)).unwrap(); - let v: Value = serde_json::from_str(&ex(t)).unwrap(); - let top: Vec<(&str, Value)> = ["template", "title", "summary"].iter().filter_map(|k| v.get(*k).map(|x| (*k, x.clone()))).collect(); + for (t, text) in examples() { + let p = parse(Path::new("/p"), &text).unwrap(); + let v: Value = serde_json::from_str(&text).unwrap(); + let top: Vec<(&str, Value)> = v.as_object().unwrap().iter().filter(|(k, _)| *k != "dev").map(|(k, x)| (k.as_str(), x.clone())).collect(); let dev: Vec<(&str, Value)> = v.get("dev").and_then(Value::as_object).map(|d| d.iter().map(|(k, x)| (k.as_str(), x.clone())).collect()).unwrap_or_default(); - assert_eq!(render(&top, &dev), ex(t), "{t}"); - assert_eq!(p.template.as_deref(), Some(t)); + assert_eq!(render(&top, &dev), text, "{t}"); + assert_eq!(renamed(&text, &t).unwrap(), text, "{t}"); + assert_eq!(p.template.as_deref(), Some(t.as_str())); } } + /// Every template's file is of this version, names its schema, and is named as its template + /// is: the project `wai create` makes from it is named for its directory instead. + #[test] + fn every_templates_file_names_its_schema_its_version_and_its_name() { + for (t, text) in examples() { + let v: Value = serde_json::from_str(&text).unwrap(); + assert_eq!((v["$schema"].as_str(), v["wai"].as_u64(), v["name"].as_str()), (Some(SCHEMA), Some(VERSION), Some(t.as_str())), "{t}"); + let mine = renamed(&text, "my-project").unwrap(); + assert_eq!(parse(Path::new("/p"), &mine).unwrap().name.as_deref(), Some("my-project"), "{t}"); + assert_eq!(mine.replace("\"my-project\"", &format!("{t:?}")), text, "{t}: only the name changes"); + } + // A file without a name gains one after its schema and version. + assert_eq!(renamed(r#"{"wai":1,"title":"x"}"#, "n").unwrap(), "{\n \"wai\": 1,\n \"name\": \"n\",\n \"title\": \"x\"\n}\n"); + assert_eq!(name_for(Path::new("/x/My Room"), "video-room"), "video-room"); + assert_eq!(name_for(Path::new("/x/my-room"), "video-room"), "my-room"); + assert_eq!(name_for(Path::new("."), "video-room"), "video-room"); + } + + #[test] + fn the_version_the_schema_and_the_name_are_held() { + let ok = |t: &str| parse(Path::new("/p"), t); + assert!(ok(&format!(r#"{{"$schema":"{SCHEMA}","wai":1,"name":"my-room"}}"#)).is_ok()); + assert!(ok("{}").is_ok(), "a file without a version is of version 1"); + assert!(ok(r#"{"wai":2}"#).unwrap_err().contains("upgrade wai")); + assert_eq!(SCHEMA, format!("{SCHEMA_OF}{VERSION}")); + // A later version adds members: the file is refused as that version, by its `wai` or by + // its `$schema`, before any member it adds is read as unknown. + for later in [ + r#"{"wai":2,"requires":{}}"#, + r#"{"requires":{},"wai":2}"#, + r#"{"$schema":"https://wai.transaction.science/schema/wai.json/2","targets":{}}"#, + r#"{"$schema":"https://wai.transaction.science/schema/wai.json/2"}"#, + r#"{"$schema":"https://wai.transaction.science/schema/wai.json/2","wai":1}"#, + ] { + assert!(ok(later).unwrap_err().contains("upgrade wai"), "{later}: {:?}", ok(later)); + } + assert!(ok(r#"{"wai":1,"requires":{}}"#).unwrap_err().contains("requires: not a member")); + assert!(ok(r#"{"$schema":"https://wai.transaction.science/schema/wai.json/0"}"#).unwrap_err().contains("$schema")); + assert!(ok(r#"{"wai":0}"#).unwrap_err().contains("version")); + assert!(ok(r#"{"wai":"1"}"#).unwrap_err().contains("version")); + assert!(ok(r#"{"$schema":"https://example.org/wai.json"}"#).unwrap_err().contains("$schema")); + for bad in ["My-Room", "my room", "-room", "room-", "9lives", "", &"a".repeat(64)] { + assert!(!valid_name(bad), "{bad:?}"); + assert!(ok(&serde_json::json!({ "name": bad }).to_string()).unwrap_err().contains("name"), "{bad:?}"); + } + for good in ["a", "my-room", "room2", "a-b-c", &"a".repeat(63)] { + assert!(valid_name(good), "{good:?}"); + } + } + + /// Two JSON values as JSON Schema 2020-12 compares them (`const`, `enum`): numbers by their + /// value, so `1` and `1.0` are one, and anything else as written. + fn same(a: &Value, b: &Value) -> bool { + match (a, b) { + (Value::Number(_), Value::Number(_)) => order(a, b) == Some(std::cmp::Ordering::Equal), + _ => a == b, + } + } + + /// Two numbers in order by their value: as integers when both are, else as doubles. + fn order(a: &Value, b: &Value) -> Option { + match (a.as_u64(), b.as_u64(), a.as_i64(), b.as_i64()) { + (Some(x), Some(y), _, _) => Some(x.cmp(&y)), + (_, _, Some(x), Some(y)) => Some(x.cmp(&y)), + _ => a.as_f64()?.partial_cmp(&b.as_f64()?), + } + } + + /// A validator of the JSON Schema keywords the committed schema uses, with 2020-12's meaning + /// of each (an integer is a number with no fraction, however written; a pattern is a regular + /// expression, run), refusing any other keyword so that a schema it cannot read fails here + /// instead of passing. + fn valid(schema: &Value, doc: &Value) -> bool { + use std::cmp::Ordering::{Greater, Less}; + let s = schema.as_object().expect("a schema is an object"); + for (k, x) in s { + let holds = match k.as_str() { + "$schema" | "$id" | "title" | "description" | "$comment" | "examples" => true, + "type" => match x.as_str().unwrap() { + "object" => doc.is_object(), + "string" => doc.is_string(), + "integer" => doc.as_f64().is_some_and(|f| f.fract() == 0.0), + "boolean" => doc.is_boolean(), + other => panic!("type {other}"), + }, + "const" => same(doc, x), + "enum" => x.as_array().unwrap().iter().any(|e| same(doc, e)), + "minimum" => !doc.is_number() || order(doc, x) != Some(Less), + "maximum" => !doc.is_number() || order(doc, x) != Some(Greater), + "pattern" => doc.as_str().is_none_or(|d| regex::Regex::new(x.as_str().unwrap()).unwrap().is_match(d)), + "required" => x.as_array().unwrap().iter().all(|r| doc.get(r.as_str().unwrap()).is_some()), + "properties" => x.as_object().unwrap().iter().all(|(p, sub)| doc.get(p).is_none_or(|d| valid(sub, d))), + "additionalProperties" => { + assert_eq!(x, &Value::Bool(false)); + let props = s["properties"].as_object().unwrap(); + doc.as_object().is_none_or(|o| o.keys().all(|k| props.contains_key(k))) + } + "if" => !valid(x, doc) || s.get("then").is_none_or(|t| valid(t, doc)), + "then" => true, + other => panic!("the schema uses {other}, which this test's validator does not read"), + }; + if !holds { + return false; + } + } + true + } + + /// The committed schema states what this reader reads: its `$id` and `$schema` are + /// [`SCHEMA`], its members and the `dev` block's are [`MEMBERS`] and [`DEV_MEMBERS`] with no + /// other, its modes are [`MODES`], and the schema and the reader accept and refuse the same + /// files, every template's among them. + #[test] + fn the_schema_states_what_the_reader_reads() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../schema/wai-json-1.schema.json"); + let schema: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(schema["$id"], SCHEMA); + assert_eq!(schema["properties"]["$schema"]["const"], SCHEMA); + assert_eq!(schema["properties"]["wai"]["const"], VERSION); + assert_eq!(schema["properties"]["name"]["pattern"], NAME_PATTERN); + let keys = |v: &Value| v["properties"].as_object().unwrap().keys().cloned().collect::>(); + assert_eq!(keys(&schema), MEMBERS); + assert_eq!(keys(&schema["properties"]["dev"]), DEV_MEMBERS); + assert_eq!(schema["properties"]["dev"]["properties"]["mode"]["enum"], serde_json::json!(MODES)); + let mut cases: Vec = examples().into_iter().map(|(_, t)| t).collect(); + cases.extend( + [ + "{}", + r#"{"wai":2}"#, + r#"{"wai":2,"requires":{}}"#, + r#"{"$schema":"https://wai.transaction.science/schema/wai.json/2","targets":{}}"#, + r#"{"wai":0}"#, + r#"{"$schema":"https://example.org/wai.json"}"#, + r#"{"name":"My-Room"}"#, + r#"{"name":"room-"}"#, + r#"{"name":"9lives"}"#, + r#"{"title":3}"#, + r#"{"extra":1}"#, + r#"{"dev":{"sites":"."}}"#, + r#"{"dev":{"mode":"stage"}}"#, + r#"{"dev":{"publishers":"4"}}"#, + r#"{"dev":{"audience":2}}"#, + r#"{"dev":{"mode":"room","audience":2}}"#, + r#"{"dev":{"mode":"show","audience":2}}"#, + r#"{"dev":{"mode":"show","audience":0}}"#, + r#"{"dev":{"mode":"show","audience":256}}"#, + r#"{"dev":{"messaging":"yes"}}"#, + // Whole numbers however written, and past the range the file holds. + r#"{"wai":1.0}"#, + r#"{"wai":1e0}"#, + r#"{"wai":2.0}"#, + r#"{"wai":1.5}"#, + r#"{"wai":-1}"#, + r#"{"dev":{"publishers":4.0}}"#, + r#"{"dev":{"publishers":4.5}}"#, + r#"{"dev":{"publishers":-1}}"#, + r#"{"dev":{"publishers":4294967295}}"#, + r#"{"dev":{"publishers":4294967296}}"#, + r#"{"dev":{"watch_ms":2e4}}"#, + r#"{"dev":{"watch_ms":9007199254740991}}"#, + r#"{"dev":{"watch_ms":9007199254740992}}"#, + r#"{"dev":{"watch_ms":18446744073709551615}}"#, + r#"{"dev":{"watch_ms":18446744073709551616}}"#, + r#"{"dev":{"mode":"show","audience":2.0}}"#, + r#"{"dev":{"mode":"show","audience":255.0}}"#, + r#"{"dev":{"mode":"show","audience":255.5}}"#, + ] + .map(String::from), + ); + cases.push(serde_json::json!({ "name": "a".repeat(64) }).to_string()); + cases.push(serde_json::json!({ "name": "a".repeat(63) }).to_string()); + for c in cases { + let doc: Value = serde_json::from_str(&c).unwrap(); + assert_eq!(valid(&schema, &doc), parse(Path::new("/p"), &c).is_ok(), "the schema and the reader differ on {c}"); + } + // The two read one number alike however it is written. + let p = parse(Path::new("/p"), r#"{"wai":1.0,"dev":{"mode":"show","publishers":4.0,"watch_ms":2e4,"audience":2.0}}"#).unwrap(); + assert_eq!((p.dev.publishers, p.dev.watch_ms, p.dev.audience), (Some(4), Some(20000), Some(2))); + assert_eq!(schema["properties"]["dev"]["properties"]["watch_ms"]["maximum"], WHOLE_MAX); + } + + /// The schema's name pattern, run as the regular expression it is, and [`valid_name`] accept + /// the same names: every name of up to four characters over letters, digits, a hyphen and + /// characters outside the rule, and names at and past the length bound. + #[test] + fn the_name_pattern_and_the_name_rule_accept_the_same_names() { + let pattern = regex::Regex::new(NAME_PATTERN).unwrap(); + let alphabet = ['a', 'z', '0', '9', '-', 'A', '_', ' ', '.', 'é']; + let mut names = vec![String::new()]; + let mut last = vec![String::new()]; + for _ in 0..4 { + last = last.iter().flat_map(|n| alphabet.iter().map(move |c| format!("{n}{c}"))).collect(); + names.extend(last.iter().cloned()); + } + for n in [62, 63, 64] { + names.push("a".repeat(n)); + names.push(format!("a{}b", "-".repeat(n - 2))); + names.push(format!("a{}", "-".repeat(n - 1))); + names.push(format!("9{}", "a".repeat(n - 1))); + } + names.extend(["my-room", "room2", "a-b-c", "My-Room", "my room", "-room", "room-", "9lives", "a\n", "a\nb"].map(String::from)); + let mut accepted = 0; + for n in &names { + assert_eq!(pattern.is_match(n), valid_name(n), "{n:?}"); + accepted += usize::from(valid_name(n)); + } + assert!(accepted > 100 && accepted < names.len(), "both kinds of name tried: {accepted} of {}", names.len()); + } + #[test] fn an_apps_file_names_its_program_and_a_show_its_audience() { let p = parse(Path::new("/p"), &render(&[("title", "x".into()), ("app", "app.oui".into())], &[("site", ".".into()), ("mode", "show".into()), ("audience", 2.into())])).unwrap(); diff --git a/wai/wai-rs/src/bin/wai_cli/dx/templates.rs b/wai/wai-rs/src/bin/wai_cli/dx/templates.rs index 2d8daef04..a9a8d6078 100644 --- a/wai/wai-rs/src/bin/wai_cli/dx/templates.rs +++ b/wai/wai-rs/src/bin/wai_cli/dx/templates.rs @@ -1,5 +1,6 @@ //! The templates `wai create` writes: each example of `wai/examples/` but its `test/` -//! directory, its files held in this binary. `tests/cli_dx.rs` holds this table to the +//! directory, its files held in this binary. Each carries an AGENTS.md: what the project is, +//! and how to build, run, test and conform it, for a coding agent. `tests/cli_dx.rs` holds this table to the //! examples on disk, file for file and byte for byte, so the two cannot drift. Every //! template is Apache-2.0 and carries the license's text as `LICENSE`: one copy in this //! binary, `wai/LICENSE`, which each example's `LICENSE` must equal byte for byte. @@ -22,6 +23,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("video-room", ".gitignore"), + ex!("video-room", "AGENTS.md"), ex!("video-room", "README.md"), ex!("video-room", "index.html"), ex!("video-room", "wai.json"), @@ -34,6 +36,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("watch-together", ".gitignore"), + ex!("watch-together", "AGENTS.md"), ex!("watch-together", "README.md"), ex!("watch-together", "index.html"), ex!("watch-together", "wai.json"), @@ -46,6 +49,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("messaging", ".gitignore"), + ex!("messaging", "AGENTS.md"), ex!("messaging", "README.md"), ex!("messaging", "index.html"), ex!("messaging", "wai.json"), @@ -58,6 +62,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("encoder-ingest", ".gitignore"), + ex!("encoder-ingest", "AGENTS.md"), ex!("encoder-ingest", "Cargo.toml"), ex!("encoder-ingest", "README.md"), ex!("encoder-ingest", "src/main.rs"), @@ -71,6 +76,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("package-and-deliver", ".gitignore"), + ex!("package-and-deliver", "AGENTS.md"), ex!("package-and-deliver", "README.md"), ex!("package-and-deliver", "floor/v.obu"), ex!("package-and-deliver", "index.html"), @@ -88,6 +94,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("verify-receipts", ".gitignore"), + ex!("verify-receipts", "AGENTS.md"), ex!("verify-receipts", "README.md"), ex!("verify-receipts", "python/_blake3.py"), ex!("verify-receipts", "python/_ed25519.py"), @@ -104,6 +111,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("apple-app", ".gitignore"), + ex!("apple-app", "AGENTS.md"), ex!("apple-app", "Package.swift"), ex!("apple-app", "README.md"), ex!("apple-app", "Sources/WaiApp/PlayerApp.swift"), @@ -120,6 +128,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("android-app", ".gitignore"), + ex!("android-app", "AGENTS.md"), ex!("android-app", "README.md"), ex!("android-app", "replay/build.gradle.kts"), ex!("android-app", "replay/src/main/kotlin/Replay.kt"), @@ -135,6 +144,7 @@ pub const TEMPLATES: &[Template] = &[ files: &[ LICENSE, ex!("remote-desktop", ".gitignore"), + ex!("remote-desktop", "AGENTS.md"), ex!("remote-desktop", "README.md"), ex!("remote-desktop", "host.sh"), ex!("remote-desktop", "wai.json"), diff --git a/wai/wai-rs/src/bin/wai_cli/mcp/mod.rs b/wai/wai-rs/src/bin/wai_cli/mcp/mod.rs index f28dacbc5..78ea3e7a4 100644 --- a/wai/wai-rs/src/bin/wai_cli/mcp/mod.rs +++ b/wai/wai-rs/src/bin/wai_cli/mcp/mod.rs @@ -1,7 +1,27 @@ //! `wai mcp` (catalogue `mcp.start`): WAI's operations as tools for a coding assistant, served -//! over the Model Context Protocol (revision 2025-06-18) on standard input and output, one -//! JSON-RPC message per line. Standard output carries the protocol alone; anything else goes to -//! standard error. +//! over the Model Context Protocol on standard input and output, one JSON-RPC message per line. +//! Standard output carries the protocol alone; anything else goes to standard error. +//! +//! It speaks both eras of the protocol, chosen by how each request opens: +//! +//! - **2026-07-28**, stateless: there is no handshake. Every request names its revision and the +//! client's capabilities in `_meta` (`io.modelcontextprotocol/protocolVersion` and +//! `io.modelcontextprotocol/clientCapabilities`, both required, or the request is refused +//! `-32602`), and is served from what it carries alone. A revision this server does not serve +//! per request is refused `-32022` (`UnsupportedProtocolVersion`) with the revisions it does in +//! `data.supported`, never the one asked for, and the revisions `initialize` negotiates in +//! `data.initialize`: a client that retries with one of `supported`, as that error asks, is +//! served, and one of the earlier era opens with `initialize` instead. +//! `server/discover` states them, the server's capabilities and its instructions. Every result +//! carries `resultType` and the server's identity in its `_meta`; `server/discover` and +//! `tools/list` carry `ttlMs` and `cacheScope`. `ping` is not a method of this revision. +//! - **2025-11-25, 2025-06-18, 2025-03-26 and 2024-11-05**, negotiated by `initialize`, as +//! before: a request without the revision in its `_meta` is served under them. 2025-11-25, the +//! last revision with a handshake, changed nothing these tools use: their names keep its +//! naming rule, their input schemas are JSON Schema 2020-12, its default dialect, a tool's own +//! error (bad input among them) is a result with `isError`, as it asks, and the server declares +//! none of the capabilities it added (tasks), so a client of it is answered in it rather than +//! moved down to 2025-06-18. //! //! A tool is an operation of the toolchain's catalogue, named by it: the operation's name with //! `_` for the dot (`receipt.verify` is `receipt_verify`), never a name of its own. The tools @@ -46,8 +66,33 @@ mod dev_room; #[path = "tools.rs"] mod tools; -/// The protocol revisions this server speaks, newest first. -const PROTOCOLS: &[&str] = &["2025-06-18", "2025-03-26", "2024-11-05"]; +/// The stateless revisions this server speaks, newest first: each request names one in its +/// `_meta` ([`META_VERSION`]). +const MODERN: &[&str] = &["2026-07-28"]; +/// The revisions an `initialize` request negotiates, newest first: one it does not know is +/// answered in the first. +const LEGACY: &[&str] = &["2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05"]; + +/// The `_meta` keys of a stateless request and result (MCP 2026-07-28, *General fields*). +const META_VERSION: &str = "io.modelcontextprotocol/protocolVersion"; +const META_CAPABILITIES: &str = "io.modelcontextprotocol/clientCapabilities"; +const META_SERVER: &str = "io.modelcontextprotocol/serverInfo"; +/// `UnsupportedProtocolVersion` (MCP 2026-07-28, *Error codes*). +const UNSUPPORTED_VERSION: i64 = -32022; +/// How long a client may hold `server/discover`'s and `tools/list`'s answers: what they state is +/// fixed for the process's life, by the options it was started with. +const TTL_MS: u64 = 3_600_000; + +/// Every revision this server speaks, newest first: the stateless ones, then those `initialize` +/// negotiates. +fn supported() -> Vec<&'static str> { + MODERN.iter().chain(LEGACY).copied().collect() +} + +/// The server's name and version, as `initialize` and every stateless result give them. +fn server_info() -> Value { + json!({"name": "wai", "title": "WAI", "version": env!("CARGO_PKG_VERSION")}) +} /// What the server tells an assistant when it connects, a sentence for each toolset it offers. /// It names no tool: the tools' table is the one place their names are written. @@ -149,7 +194,7 @@ impl Server { /// One JSON-RPC message's answer; `None` for a notification. pub(crate) fn handle(&mut self, msg: &Value) -> Option { if msg.is_array() { - return Some(json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "a batch is not taken (MCP 2025-06-18): send one message at a time"}})); + return Some(json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "a batch is not taken (MCP 2025-06-18 and 2026-07-28): send one message at a time"}})); } // A response or a notification from the client (no id): nothing to answer. let id = msg.get("id").cloned()?; @@ -159,37 +204,104 @@ impl Server { return None; } let ok = |result: Value| Some(json!({"jsonrpc": "2.0", "id": id, "result": result})); - let err = |code: i64, message: String| Some(json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message}})); + let err = |code: i64, message: String, data: Option| { + let mut e = json!({"code": code, "message": message}); + if let Some(d) = data { + e["data"] = d; + } + Some(json!({"jsonrpc": "2.0", "id": id, "error": e})) + }; + // `initialize` opens a legacy session; a request whose `_meta` names a revision is a + // stateless one; any other is served under the legacy revisions, as before. + let meta = params.get("_meta"); + if method != "initialize" + && let Some(asked) = meta.and_then(|m| m.get(META_VERSION)) + { + return match self.stateless(method, ¶ms, asked) { + Ok(mut result) => { + result["resultType"] = json!("complete"); + let mut m = serde_json::Map::new(); + m.insert(META_SERVER.into(), server_info()); + result["_meta"] = Value::Object(m); + ok(result) + } + Err((code, message, data)) => err(code, message, data), + }; + } match method { "initialize" => { self.client = params.pointer("/clientInfo/name").and_then(Value::as_str).map(String::from); - let asked = params.get("protocolVersion").and_then(Value::as_str).unwrap_or(PROTOCOLS[0]); - let version = PROTOCOLS.iter().find(|v| **v == asked).copied().unwrap_or(PROTOCOLS[0]); + let asked = params.get("protocolVersion").and_then(Value::as_str).unwrap_or(LEGACY[0]); + let version = LEGACY.iter().find(|v| **v == asked).copied().unwrap_or(LEGACY[0]); ok(json!({ "protocolVersion": version, "capabilities": {"tools": {"listChanged": false}}, - "serverInfo": {"name": "wai", "title": "WAI", "version": env!("CARGO_PKG_VERSION")}, + "serverInfo": server_info(), "instructions": instructions(&self.toolsets), })) } "ping" => ok(json!({})), "tools/list" => ok(json!({"tools": tools::list(self)})), - "tools/call" => { - let name = params.get("name").and_then(Value::as_str).unwrap_or(""); - let args = params.get("arguments").cloned().unwrap_or(json!({})); - if !args.is_object() { - return err(-32602, "arguments: an object".into()); - } - match tools::call(self, name, &args) { - None => err(-32602, format!("no tool {name}; tools/list names them")), - Some(Ok(v)) => { - let text = serde_json::to_string_pretty(&v).unwrap_or_default(); - ok(json!({"content": [{"type": "text", "text": text}], "structuredContent": v, "isError": false})) - } - Some(Err(e)) => ok(json!({"content": [{"type": "text", "text": e}], "isError": true})), - } + "tools/call" => match self.call(¶ms) { + Ok(r) => ok(r), + Err(m) => err(-32602, m, None), + }, + "server/discover" => err(-32602, format!("server/discover is a request of MCP 2026-07-28: name the revision in its _meta ({META_VERSION}), or open with initialize for {}", LEGACY.join(", ")), None), + other => err(-32601, format!("no method {other}"), None), + } + } + + /// A request of a stateless revision (MCP 2026-07-28), served from what it carries alone: its + /// result without `resultType` and `_meta`, which [`Server::handle`] adds, or its error's + /// code, message and data. + fn stateless(&mut self, method: &str, params: &Value, asked: &Value) -> Result)> { + let Some(version) = asked.as_str() else { + return Err((-32602, format!("_meta: {META_VERSION} is a string"), None)); + }; + if !MODERN.contains(&version) { + let why = if LEGACY.contains(&version) { + format!("unsupported protocol version: {version} is negotiated by initialize ({}), not named in a request's _meta ({})", LEGACY.join(", "), MODERN.join(", ")) + } else { + format!("unsupported protocol version {version}: a request names {}, or initialize negotiates {}", MODERN.join(", "), LEGACY.join(", ")) + }; + // `supported` is what a request may name, which the one asked for is not, so a client + // that retries with one of them is served; the earlier era's go apart, by name. + return Err((UNSUPPORTED_VERSION, why, Some(json!({"supported": MODERN, "requested": version, "initialize": LEGACY})))); + } + if !params.pointer("/_meta").and_then(|m| m.get(META_CAPABILITIES)).is_some_and(Value::is_object) { + return Err((-32602, format!("_meta: {META_CAPABILITIES} (an object) is required on every request of MCP {version}"), None)); + } + match method { + "server/discover" => Ok(json!({ + "supportedVersions": supported(), + "capabilities": {"tools": {}}, + "instructions": instructions(&self.toolsets), + "ttlMs": TTL_MS, + "cacheScope": "public", + })), + "tools/list" => Ok(json!({"tools": tools::list(self), "ttlMs": TTL_MS, "cacheScope": "public"})), + "tools/call" => self.call(params).map_err(|m| (-32602, m, None)), + "ping" => Err((-32601, format!("no method ping in MCP {version}, which removed it"), None)), + other => Err((-32601, format!("no method {other}"), None)), + } + } + + /// `tools/call`'s result, the same in every revision: the tool's structured result and its + /// text, or its error as text with `isError`. `Err` for a call that names no tool offered or + /// whose arguments are not an object. + fn call(&mut self, params: &Value) -> Result { + let name = params.get("name").and_then(Value::as_str).unwrap_or(""); + let args = params.get("arguments").cloned().unwrap_or(json!({})); + if !args.is_object() { + return Err("arguments: an object".into()); + } + match tools::call(self, name, &args) { + None => Err(format!("no tool {name}; tools/list names them")), + Some(Ok(v)) => { + let text = serde_json::to_string_pretty(&v).unwrap_or_default(); + Ok(json!({"content": [{"type": "text", "text": text}], "structuredContent": v, "isError": false})) } - other => err(-32601, format!("no method {other}")), + Some(Err(e)) => Ok(json!({"content": [{"type": "text", "text": e}], "isError": true})), } } } diff --git a/wai/wai-rs/src/catalogue.rs b/wai/wai-rs/src/catalogue.rs index b692b68ed..467281870 100644 --- a/wai/wai-rs/src/catalogue.rs +++ b/wai/wai-rs/src/catalogue.rs @@ -2479,6 +2479,64 @@ mod tests { assert!(f.is_empty(), "{}", f.join("\n")); } + /// The closed verb list is written in three places: [`VERBS`], the catalogue's README (with + /// the record of the two verbs that joined it after its approval, upscale and tonemap), and + /// the doors' check, which holds a door's name to the catalogue only when its verb is listed. + /// Each states exactly [`VERBS`]. So is the near-synonym list, which the README states beside + /// it and the doors' check refuses a door's name by: each states exactly [`NEAR_SYNONYMS`], + /// each near-synonym with the verb to use (the first word of the catalogue's). + #[test] + fn every_text_that_states_the_verb_list_states_the_same_list() { + use std::collections::BTreeSet; + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let read = |rel: &str| std::fs::read_to_string(root.join(rel)).unwrap_or_else(|e| panic!("{rel}: {e}")); + let want: BTreeSet<&str> = VERBS.iter().copied().collect(); + assert_eq!(want.len(), VERBS.len(), "VERBS lists a verb twice"); + let between = |text: &'static str, open: &str, close: char| -> BTreeSet<&'static str> { + let at = text.find(open).unwrap_or_else(|| panic!("no {open:?}")) + open.len(); + let body = &text[at..at + text[at..].find(close).unwrap()]; + body.split(',').map(|w| w.trim().trim_matches('"')).filter(|w| !w.is_empty()).collect() + }; + let readme: &'static str = read("wai/catalogue/README.md").leak(); + assert_eq!(between(readme, "rule N3's closed list (", ')'), want, "wai/catalogue/README.md"); + for v in ["upscale", "tonemap"] { + assert!(readme.contains(&format!("`{v}`")), "wai/catalogue/README.md records why {v} joined the list"); + } + let script: &'static str = read("scripts/check-wai-ops-doors.py").leak(); + assert_eq!(between(script, "\nVERBS = {", '}'), want, "scripts/check-wai-ops-doors.py"); + + // The near-synonyms, each with its verb. + let near: std::collections::BTreeMap<&str, &str> = + NEAR_SYNONYMS.iter().map(|(n, v)| (*n, v.split_whitespace().next().unwrap())).collect(); + assert_eq!(near.len(), NEAR_SYNONYMS.len(), "NEAR_SYNONYMS lists a word twice"); + for (n, v) in &near { + assert!(VERBS.contains(v) && !VERBS.contains(n), "{n} is a near-synonym of the verb {v}"); + } + // The script's NEAR: "word": "verb" pairs. + let pairs: std::collections::BTreeMap<&str, &str> = between(script, "\nNEAR = {", '}') + .into_iter() + .map(|p| { + let (n, v) = p.split_once(':').unwrap_or_else(|| panic!("NEAR: {p:?} is not a pair")); + (n.trim().trim_matches('"'), v.trim().trim_matches('"')) + }) + .collect(); + assert_eq!(pairs, near, "scripts/check-wai-ops-doors.py's NEAR"); + // The README's sentence: clauses "`a`, `b` and `c` are `v`", one for each verb. + let open = "never a near-synonym ("; + let at = readme.find(open).unwrap_or_else(|| panic!("the README states no near-synonyms")) + open.len(); + let sentence = &readme[at..at + readme[at..].find(')').unwrap()]; + let ticked = |t: &'static str| -> Vec<&'static str> { t.split('`').skip(1).step_by(2).collect() }; + let mut stated = std::collections::BTreeMap::new(); + for clause in sentence.split(';') { + let (words, verb) = clause.split_once(" are ").or_else(|| clause.split_once(" is ")).unwrap_or_else(|| panic!("{clause:?}")); + let verb = ticked(verb); + for w in ticked(words) { + stated.insert(w, verb[0]); + } + } + assert_eq!(stated, near, "wai/catalogue/README.md's near-synonyms"); + } + #[test] fn the_rules_refuse_what_they_name() { // Each rule, broken in a copy of the table, is a finding that names it. diff --git a/wai/wai-rs/src/codecs/registry.rs b/wai/wai-rs/src/codecs/registry.rs index b3bc263f2..841e2168f 100644 --- a/wai/wai-rs/src/codecs/registry.rs +++ b/wai/wai-rs/src/codecs/registry.rs @@ -646,6 +646,7 @@ pub struct WireIdentifier { const HTTP_DELIVERY: &str = "extensions/http-delivery.md"; const MOQ_FORMAT: &str = "extensions/moq-streaming-format.md"; const JWP_RECEIPTS: &str = "extensions/jwp-receipts.md"; +const HOLDINGS: &str = "extensions/holdings.md"; const PACKAGING: &str = "extensions/packaging.md"; const LIVE_EVENTS: &str = "extensions/live-events.md"; const LIVE_ROOMS: &str = "extensions/live-rooms.md"; @@ -748,6 +749,8 @@ pub const WIRE_IDENTIFIERS: &[WireIdentifier] = &[ wire("/.well-known/wai-keys", WireKind::WellKnown, JWP_RECEIPTS, "§7.1", WireStatus::Defined), wire("wai-key-status", WireKind::DocumentKind, JWP_RECEIPTS, "§7.9", WireStatus::Defined), wire("wai-module-grant", WireKind::DocumentKind, JWP_RECEIPTS, "§7.9", WireStatus::Defined), + // A node's signed statement of which objects of which groups it holds now. + wire("WHD1", WireKind::Magic, HOLDINGS, "§4.1", WireStatus::Defined), wire("WSIX", WireKind::Magic, PACKAGING, "§6", WireStatus::Defined), wire("science.transaction.wai.class", WireKind::DataId, PACKAGING, "§8", WireStatus::Defined), wire("science.transaction.wai.stages", WireKind::DataId, PACKAGING, "§8", WireStatus::Defined), diff --git a/wai/wai-rs/src/holdings.rs b/wai/wai-rs/src/holdings.rs index aa459d8bd..d8beb896f 100644 --- a/wai/wai-rs/src/holdings.rs +++ b/wai/wai-rs/src/holdings.rs @@ -391,4 +391,15 @@ mod tests { assert_eq!(who[1], vec![a.holder]); assert_eq!(origin, vec![2, 3]); } + + /// §4.1's magic is the registry's: one row names it, a magic of this text's §4.1, and no + /// other wire identifier of any kind is the same four letters in any case. + #[test] + fn the_magic_is_registered_once_and_shared_with_nothing() { + use crate::codecs::registry::{WireKind, WIRE_IDENTIFIERS}; + let m = std::str::from_utf8(MAGIC).unwrap(); + let rows: Vec<_> = WIRE_IDENTIFIERS.iter().filter(|w| w.name.eq_ignore_ascii_case(m)).collect(); + assert_eq!(rows.len(), 1, "{m}: the registry has one row for it"); + assert_eq!((rows[0].kind, rows[0].document, rows[0].section), (WireKind::Magic, "extensions/holdings.md", "§4.1")); + } } diff --git a/wai/wai-rs/tests/cli_app.rs b/wai/wai-rs/tests/cli_app.rs index 19b6d0b9f..12eac8ce4 100644 --- a/wai/wai-rs/tests/cli_app.rs +++ b/wai/wai-rs/tests/cli_app.rs @@ -97,7 +97,7 @@ fn an_export_is_an_ordinary_project_and_its_check_holds() { let wj = std::fs::read_to_string(p.join("wai.json")).unwrap(); assert_eq!( wj, - "{\n \"title\": \"Friday night\",\n \"summary\": \"A WAI app made with Create: a show, with hosts on camera and an audience that watches and votes.\",\n \"app\": \"app.oui\",\n \"dev\": { \"site\": \".\", \"mode\": \"show\", \"audience\": 2 }\n}\n" + "{\n \"$schema\": \"https://wai.transaction.science/schema/wai.json/1\",\n \"wai\": 1,\n \"name\": \"show\",\n \"title\": \"Friday night\",\n \"summary\": \"A WAI app made with Create: a show, with hosts on camera and an audience that watches and votes.\",\n \"app\": \"app.oui\",\n \"dev\": { \"site\": \".\", \"mode\": \"show\", \"audience\": 2 }\n}\n" ); // A template's project has the same shape: its project file read by the same reader and // laid out the same way, a README and the licence. diff --git a/wai/wai-rs/tests/cli_mcp.rs b/wai/wai-rs/tests/cli_mcp.rs index 21467735d..a3a8c67fe 100644 --- a/wai/wai-rs/tests/cli_mcp.rs +++ b/wai/wai-rs/tests/cli_mcp.rs @@ -3,6 +3,13 @@ //! JSON-RPC 2.0, MCP 2025-06-18): the handshake, the tool list, and every tool through //! `tools/call`. //! +//! - The stateless revision, MCP 2026-07-28, with no handshake: `server/discover` states every +//! revision the server speaks, its capabilities and its caching hints; `tools/list` and +//! `tools/call` are served from each request's `_meta` alone, every result with `resultType` +//! and the server's identity; a revision it does not serve per request is refused `-32022` with +//! the ones it does, which never hold the one asked for, and those `initialize` negotiates +//! apart; a request without the client's capabilities `-32602`, and `ping`, which that +//! revision removed, `-32601`. The same process still answers `initialize` after them. //! - Every tool is an operation of the toolchain's catalogue, named by it: the tool list is //! exactly the operations below, each tool's name its operation's with `_` for the dot, each //! operation of the catalogue (`wai::catalogue`). `core` is offered by default, @@ -41,7 +48,17 @@ struct Client { } impl Client { + /// A client that has opened a session with `initialize` (MCP 2025-06-18). fn start(project: &Path, extra: &[&str]) -> Client { + let mut c = Client::spawn(project, extra); + let init = c.request("initialize", json!({"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "cli_mcp test", "version": "1"}})); + assert_eq!(init["result"]["protocolVersion"], "2025-06-18", "{init}"); + c.send(&json!({"jsonrpc": "2.0", "method": "notifications/initialized"})); + c + } + + /// A server process, and nothing sent to it yet. + fn spawn(project: &Path, extra: &[&str]) -> Client { let mut child = Command::new(env!("CARGO_BIN_EXE_wai")) .arg("mcp") .arg("--spec-root") @@ -56,11 +73,7 @@ impl Client { .expect("wai mcp starts"); let stdin = child.stdin.take().unwrap(); let out = BufReader::new(child.stdout.take().unwrap()); - let mut c = Client { child, stdin, out, next: 1 }; - let init = c.request("initialize", json!({"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "cli_mcp test", "version": "1"}})); - assert_eq!(init["result"]["protocolVersion"], "2025-06-18", "{init}"); - c.send(&json!({"jsonrpc": "2.0", "method": "notifications/initialized"})); - c + Client { child, stdin, out, next: 1 } } fn send(&mut self, v: &Value) { @@ -174,9 +187,96 @@ fn the_handshake_and_the_tool_list() { writeln!(c.stdin, "{{not json").unwrap(); c.stdin.flush().unwrap(); assert_eq!(c.line()["error"]["code"], -32700); - // An older revision a client asks for is answered in it. - let mut old = Client::start(&d, &[]); - assert_eq!(old.request("initialize", json!({"protocolVersion": "2024-11-05"}))["result"]["protocolVersion"], "2024-11-05"); + // Each revision initialize negotiates is answered in it, from 2025-11-25, the last with a + // handshake, down; one it does not know, in the newest of them. + for (asked, answered) in [("2025-11-25", "2025-11-25"), ("2025-06-18", "2025-06-18"), ("2025-03-26", "2025-03-26"), ("2024-11-05", "2024-11-05"), ("1999-01-01", "2025-11-25")] { + let mut c = Client::spawn(&d, &[]); + let init = c.request("initialize", json!({"protocolVersion": asked, "capabilities": {}})); + assert_eq!(init["result"]["protocolVersion"], answered, "{init}"); + assert_eq!(tool_names(&mut c), names_of(&[CORE]), "{asked}: the same tools"); + } +} + +/// A request of the stateless revision: `params` with the `_meta` MCP 2026-07-28 requires on +/// every request, naming `version`. +fn stateless(version: &str, mut params: Value) -> Value { + params["_meta"] = json!({ + "io.modelcontextprotocol/protocolVersion": version, + "io.modelcontextprotocol/clientCapabilities": {}, + "io.modelcontextprotocol/clientInfo": {"name": "cli_mcp test", "version": "1"}, + }); + params +} + +#[test] +fn the_stateless_revision_is_served_with_no_handshake() { + let d = scratch("stateless"); + let mut c = Client::spawn(&d, &[]); + let every = json!(["2026-07-28", "2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05"]); + let complete = |v: &Value| { + let r = &v["result"]; + assert_eq!(r["resultType"], "complete", "{v}"); + assert_eq!(r["_meta"]["io.modelcontextprotocol/serverInfo"]["name"], "wai", "{v}"); + }; + let cacheable = |v: &Value| { + assert!(v["result"]["ttlMs"].as_u64().is_some(), "{v}: ttlMs, a whole number of milliseconds"); + assert_eq!(v["result"]["cacheScope"], "public", "{v}"); + }; + + // The probe a client of both eras sends first, before anything else. + let d0 = c.request("server/discover", stateless("2026-07-28", json!({}))); + complete(&d0); + cacheable(&d0); + assert_eq!(d0["result"]["supportedVersions"], every, "{d0}"); + assert!(d0["result"]["capabilities"]["tools"].is_object(), "{d0}"); + assert!(d0["result"]["instructions"].as_str().is_some_and(|i| i.contains("WAI")), "{d0}"); + + // The tools, listed and called from what each request carries alone. + let list = c.request("tools/list", stateless("2026-07-28", json!({}))); + complete(&list); + cacheable(&list); + let names: Vec = list["result"]["tools"].as_array().unwrap().iter().map(|t| t["name"].as_str().unwrap().to_string()).collect(); + assert_eq!(names, names_of(&[CORE])); + let call = c.request("tools/call", stateless("2026-07-28", json!({"name": "registry_show", "arguments": {"capability": "wai.image.png"}}))); + complete(&call); + assert_eq!(call["result"]["isError"], false, "{call}"); + assert_eq!(call["result"]["structuredContent"]["registry"]["capability"], "wai.image.png", "{call}"); + let bad = c.request("tools/call", stateless("2026-07-28", json!({"name": "registry_show", "arguments": {}}))); + complete(&bad); + assert_eq!(bad["result"]["isError"], true, "a tool's own error is a result: {bad}"); + assert_eq!(c.request("tools/call", stateless("2026-07-28", json!({"name": "no_such_tool", "arguments": {}})))["error"]["code"], -32602); + + // A revision it does not serve per request: refused with the ones it does, the client's + // retry rule, which never holds the one asked for, so a retry with one of them is served. A + // legacy one is negotiated by initialize, not named per request, and those are named apart. + for v in ["1900-01-01", "2025-11-25", "2025-06-18", "2024-11-05"] { + let e = c.request("tools/list", stateless(v, json!({}))); + assert_eq!(e["error"]["code"], -32022, "{e}"); + let data = &e["error"]["data"]; + assert_eq!((data["supported"].clone(), data["requested"].clone()), (json!(["2026-07-28"]), json!(v)), "{e}"); + assert_eq!(data["initialize"], json!(every.as_array().unwrap()[1..]), "{e}"); + let supported = data["supported"].as_array().unwrap(); + assert!(!supported.contains(&json!(v)), "{e}: the revision asked for is not one to retry with"); + for retry in supported { + complete(&c.request("tools/list", stateless(retry.as_str().unwrap(), json!({})))); + } + } + assert!(c.request("tools/list", stateless("2025-06-18", json!({})))["error"]["message"].as_str().unwrap().contains("initialize")); + // The client's capabilities are required on every request. + let mut no_caps = stateless("2026-07-28", json!({})); + no_caps["_meta"].as_object_mut().unwrap().remove("io.modelcontextprotocol/clientCapabilities"); + assert_eq!(c.request("tools/list", no_caps)["error"]["code"], -32602); + // ping is not a method of this revision; server/discover is not one of the legacy ones. + assert_eq!(c.request("ping", stateless("2026-07-28", json!({})))["error"]["code"], -32601); + assert_eq!(c.request("server/discover", json!({}))["error"]["code"], -32602); + + // The same process answers a legacy client too, and serves it as before. + let init = c.request("initialize", json!({"protocolVersion": "2025-06-18", "capabilities": {}})); + assert_eq!(init["result"]["protocolVersion"], "2025-06-18", "{init}"); + let old = c.request("tools/list", json!({})); + assert!(old["result"]["tools"].is_array() && old["result"].get("resultType").is_none(), "{old}"); + // And a stateless request after it is still served statelessly. + complete(&c.request("tools/list", stateless("2026-07-28", json!({})))); } #[test]