diff --git a/.github/wai-conformance/README.md b/.github/wai-conformance/README.md new file mode 100644 index 000000000..df6975db0 --- /dev/null +++ b/.github/wai-conformance/README.md @@ -0,0 +1,34 @@ +# What wai-conformance.yml runs, one file a step + +`wai-conformance.yml` keeps no hand list of its verifier steps, corpora or kernel tests. Its +lists are derived, when it runs, from the files here and from tags in the test files, by +[`scripts/wai-conformance-lists.py`](../../scripts/wai-conformance-lists.py), whose +docstring states every key; its gate reads the steps' `paths:` through +`scripts/pr-touches-paths.py`. A change that adds a verifier, a corpus or a kernel test adds +a file, or a line to a file of its own, and edits no line of the workflow, unless a step's +`paths:` names a file under none of the trigger's paths (`check` names it). Its features and +`[[test]]` entries in `wai/wai-rs/Cargo.toml` stay shared +([`wai/CONTRIBUTING.md`](../../wai/CONTRIBUTING.md)). + +- `verifiers/.sh`: one step of the `independent-verifiers` job. Its body is the step's + shell, run in `wai/` (or `working-directory:`) with `bash -eo pipefail`, as a workflow step + runs. Its header, the leading `# key: value` lines, gives `step:` (its name), `order:` (steps + run in ascending order, a tie by file name), `features:` (each list compiles alone in + `features-alone`, and every feature joins `WAI_FEATURES`), and where it needs them `alone:`, + `build: pq`, `release-features:`, `bins:`, `pq-bins:`, `paths:` (the extension texts it + reads, which the workflow's gate then runs on) and `working-directory:`. Prose comments + follow the header, after a `#` line. +- `corpora/.txt`: one cargo-class corpus, verified by its own tool on every leg of + `cargo-classes` (`corpus:`, `order:`, `bin:`, `dir:`). +- A test file of `wai/wai-rs/tests/` whose first line is `// wai-conformance: int-kernels` + runs in the fast integer kernels' steps of both profiles (`int-kernels-release`: the + release step only; `int-kernels-amd`: the AVX-512 host too). The steps' features are the + tagged tests' `required-features`. +- `baseline.json`: the lists the workflow kept by hand before they were derived. The + repository checks require every derived list to hold every entry of it, and every step it + names to run. + +To see what the workflow runs: `python3 scripts/wai-conformance-lists.py dry-run` (of another +commit: `--ref `; two of them diff line by line). To run one step as CI does: +`python3 scripts/wai-conformance-lists.py run --only `. To check the files: +`python3 scripts/wai-conformance-lists.py check`. diff --git a/.github/wai-conformance/baseline.json b/.github/wai-conformance/baseline.json new file mode 100644 index 000000000..c3dddfab8 --- /dev/null +++ b/.github/wai-conformance/baseline.json @@ -0,0 +1,908 @@ +{ + "note": "The lists wai-conformance.yml kept by hand at `commit`, before they were derived from the files of .github/wai-conformance/ and the tags of wai/wai-rs/tests/ (scripts/wai-conformance-lists.py baseline --ref ). `selftest` requires every derived list to hold every entry here and every step named here to run; `selftest --exact` requires the lists equal (with `added`, the entries the change that derived them added on purpose) and the steps' bodies the same, in order. Coverage leaves a list only by an edit of this file.", + "commit": "a223c597dae30c57da0804d06b19f5096f016db6", + "added": { + "paths": [ + ".github/wai-conformance/**", + "scripts/wai-conformance-lists.py" + ], + "why": "the files that now say what the workflow runs start it, as the workflow file itself does" + }, + "paths": [ + "wai/wai-rs/**", + "wai/examples/**", + "wai/wai-quantum/**", + "wai/wai-ops/**", + "wai/wai-app/**", + "joule-ui/joule-ui-rs/crates/joule-ui-core/**", + "joule-ui/joule-ui-rs/crates/joule-ui-lang/**", + "joule-ui/joule-ui-rs/crates/joule-ui-interaction/**", + "wai/*-conformance/**", + "wai/byte-exact-conformance/**", + "wai/registry/**", + "wai/vocab/**", + "wai/glossary/**", + "wai/SPEC.md", + "wai/REGISTRY.md", + "wai/extensions/integer-payloads.md", + "wai/extensions/staged-delivery.md", + "wai/extensions/http-delivery.md", + "wai/extensions/moq-streaming-format.md", + "wai/extensions/jwp-receipts.md", + "wai/extensions/quantum-sim.md", + "wai/extensions/live-events.md", + "wai/extensions/live-rooms.md", + "wai/extensions/capture-binding.md", + "wai/extensions/live-compose.md", + "wai/extensions/live-recording.md", + "wai/extensions/timed-text.md", + "wai/extensions/access-services.md", + "wai/extensions/live-metering.md", + "wai/extensions/energy-steering.md", + "wai/extensions/live-moderation.md", + "wai/extensions/forensic-watermark.md", + "wai/extensions/confidentiality.md", + "wai/extensions/live-messaging.md", + "wai/extensions/live-ads.md", + "wai/extensions/ad-presentation.md", + "wai/extensions/key-release.md", + "wai/extensions/conformance-results.md", + "joule-code/joule-code-rs/crates/joule-moq-web/web/ads/**", + "wai/extensions/stitched-presentation.md", + "wai/extensions/channel-lineup.md", + "wai/extensions/live-provenance.md", + "wai/extensions/deriving-edge.md", + "wai/extensions/synthesis.md", + "wai/extensions/exact-upscale.md", + "wai/extensions/prior-carriage.md", + "wai/extensions/scene-carriage.md", + "wai/extensions/avatars.md", + "wai/tools/wai_avatar_face_train.py", + "wai/extensions/ai-usage.md", + "wai/extensions/contribution.md", + "wai/extensions/live-contribution.md", + "wai/extensions/settlement.md", + "wai/extensions/receipt-ledger.md", + "wai/extensions/revenue-share.md", + "wai/extensions/opportunity-report.md", + "wai/extensions/stream-audit.md", + "wai/extensions/fleet.md", + "wai/extensions/route-record.md", + "wai/extensions/native-sink.md", + "wai/extensions/browser-sink.md", + "wai/extensions/capability-constraints.md", + "wai/extensions/sink-energy.md", + "wai/wai-live-web/**", + "wai/extensions/takedown.md", + "wai/extensions/co-performance.md", + "wai/extensions/age-assurance.md", + "wai/extensions/influence.md", + "wai/extensions/direction.md", + "wai/extensions/bearer-receipts.md", + "wai/extensions/carousel.md", + "wai/extensions/holdings.md", + "wai/extensions/exploration.md", + "wai/extensions/platform-encode.md", + "wai/extensions/co-watch.md", + "wai/extensions/live-worlds.md", + "joule-code/joule-code-rs/crates/joule-moq-web/web/room/cowatch*", + "wai/extensions/remote-desktop.md", + "joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote*", + "wai/extensions/opus-dred.md", + "wai/extensions/telemetry-aggregation.md", + "wai/extensions/delivery-telemetry.md", + "wai/extensions/exact-tonemap.md", + "wai/extensions/robustness-evidence.md", + "wai/extensions/splat-exact.md", + "wai/extensions/gltf/**", + "wai/tools/wai_ops_check.py", + "wai/extensions/pq-signatures.md", + "wai/extensions/format-probes.md", + "wai/extensions/media-identify.md", + "wai/extensions/projection.md", + "wai/extensions/loc-carriage.md", + "wai/research/determinism-cost.md", + "wai/research/determinism-cost/**", + "wai/tools/wai_determinism_cost_verify.py", + "wai/extensions/determinism-tiers.md", + "wai/research/catalogue-prior.md", + "wai/research/catalogue-prior/**", + "wai/tools/wai_catalogue_prior_verify.py", + "wai/extensions/stream-termination.md", + "wai/wai-web/**", + "wai/tools/wai_audio_provenance_check.py", + ".github/workflows/wai-conformance.yml" + ], + "WAI_FEATURES": [ + "key_discovery", + "http", + "video_manifest", + "stage_measure", + "live_events", + "live_room", + "capture_take", + "live_compose", + "live_record", + "timed_text", + "access_services", + "live_metering", + "energy_steering", + "live_moderation", + "dred_exact", + "live_ads", + "live_provenance", + "synthesis", + "synthesis_mark", + "neural_int", + "exact_upscale", + "robustness", + "ai_usage", + "contribution", + "settlement", + "browser_sink", + "receipt_claims", + "takedown", + "influence", + "direction", + "co_performance", + "age_assurance", + "bearer_receipts", + "exploration", + "co_watch", + "telemetry_aggregate", + "exact_tonemap", + "splat_exact", + "live_watermark", + "derive_edge", + "sink_energy", + "key_release", + "stitched_presentation", + "prior_wire", + "scene_map", + "avatar_face", + "receipt_ledger", + "stream_audit", + "fleet", + "native_sink", + "platform_encode", + "remote_desktop", + "stream_termination", + "live_world", + "format_probe", + "identify", + "live_messaging", + "conform_result", + "revenue_share", + "opportunity_report", + "channel_lineup", + "route_record", + "carousel", + "holdings" + ], + "WAI_PQ_FEATURES": [ + "synthesis", + "pq_signatures" + ], + "WAI_RELEASE_FEATURES": [ + "dred_exact", + "synthesis_mark", + "robustness", + "avatar_face" + ], + "union_bins": [ + "wai_http_vectors", + "wai_stage_measure", + "wai_browser_sink_vectors", + "wai_claim_vectors", + "wai_jwp_vectors", + "wai_signer_vectors", + "wai_sink_energy_vectors", + "wai_stitched_presentation", + "wai_stream_termination_vectors", + "wai_format_probe", + "wai_identify_vectors", + "wai_conform_result_vectors", + "wai_channel_lineup_vectors" + ], + "pq_bins": [ + "wai_pq_vectors", + "wai_claim_vectors", + "wai_jwp_vectors", + "wai_signer_vectors" + ], + "union_tests": [ + "live_events", + "live_room", + "capture_take", + "live_compose", + "live_record", + "timed_text", + "access_services", + "live_metering", + "energy_steering", + "live_moderation", + "live_ads", + "ad_presentation", + "live_provenance", + "synthesis", + "exact_upscale", + "cli_derive", + "ai_usage", + "contribution", + "settlement", + "browser_sink", + "takedown", + "influence", + "direction", + "co_performance", + "age_assurance", + "bearer_receipts", + "carousel", + "holdings", + "exploration", + "co_watch", + "telemetry_aggregate", + "exact_tonemap", + "splat_exact", + "live_watermark", + "derive_edge", + "det_cost_bin", + "sink_energy", + "key_release", + "stitched_presentation", + "prior_wire", + "scene_map", + "receipt_ledger", + "stream_audit", + "fleet", + "native_sink", + "platform_encode", + "remote_desktop", + "stream_termination", + "live_world", + "format_probe", + "identify", + "live_messaging", + "conform_result", + "revenue_share", + "opportunity_report", + "channel_lineup", + "route_record" + ], + "release_tests": [ + "dred_exact", + "dred_exact_measure", + "synthesis_mark", + "robustness", + "avatar_face" + ], + "alone": [ + "key_discovery", + "http,video_manifest,key_discovery", + "stage_measure", + "live_events", + "live_room", + "capture_take", + "live_compose", + "live_record", + "timed_text", + "access_services", + "live_metering", + "energy_steering", + "live_moderation", + "dred_exact", + "live_ads", + "live_provenance", + "synthesis", + "synthesis_mark", + "neural_int", + "exact_upscale", + "robustness", + "ai_usage", + "contribution", + "settlement", + "browser_sink", + "browser_sink,receipt_claims", + "takedown", + "influence", + "direction", + "co_performance", + "age_assurance", + "bearer_receipts", + "carousel", + "holdings", + "exploration", + "co_watch", + "telemetry_aggregate", + "exact_tonemap", + "splat_exact", + "live_watermark", + "derive_edge", + "pq_signatures", + "synthesis,pq_signatures", + "sink_energy", + "key_release", + "stitched_presentation", + "prior_wire", + "prior_wire,neural_int", + "scene_map", + "avatar_face", + "receipt_ledger", + "stream_audit", + "fleet", + "native_sink", + "platform_encode", + "remote_desktop", + "stream_termination", + "live_world", + "format_probe", + "live_messaging", + "conform_result", + "revenue_share", + "opportunity_report", + "channel_lineup", + "route_record" + ], + "steps": [ + { + "step": "int_payload_verify.py", + "wd": "wai", + "body": "python3 int-payload-conformance/int_payload_verify.py int-payload-conformance" + }, + { + "step": "determinism cost · the record, the text quoting it, and wai_det_cost", + "wd": "wai", + "body": "set -euo pipefail\npython3 -I tools/wai_determinism_cost_verify.py research/determinism-cost/record.json \\\n research/determinism-cost.md extensions/determinism-tiers.md\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test det_cost_bin)" + }, + { + "step": "catalogue prior · the record and the text quoting it", + "wd": "wai", + "body": "python3 -I tools/wai_catalogue_prior_verify.py research/catalogue-prior research/catalogue-prior.md" + }, + { + "step": "staged_verify.py", + "wd": "wai", + "body": "python3 staged-conformance/staged_verify.py staged-conformance" + }, + { + "step": "http_verify.py", + "wd": "wai", + "body": "python3 http-conformance/http_verify.py http-conformance" + }, + { + "step": "The HTTP differential outside the corpus (wai-rs against http_verify.py)", + "wd": "wai", + "body": "set -euo pipefail\nfor seed in 11 12; do\n \"$RUNNER_TEMP/wai-bins/union/wai_http_vectors\" fuzz http-conformance --seed \"$seed\" --count 400 --out \"$RUNNER_TEMP/http-fuzz-$seed.json\"\n python3 http-conformance/http_verify.py cases \"$RUNNER_TEMP/http-fuzz-$seed.json\" http-conformance\ndone" + }, + { + "step": "measure_verify.py", + "wd": "wai", + "body": "python3 staged-measure-conformance/measure_verify.py staged-measure-conformance" + }, + { + "step": "sim_verify.py", + "wd": "wai", + "body": "python3 staged-measure-conformance/sim_verify.py staged-measure-conformance" + }, + { + "step": "sim_diff.py, wai_stage_measure against sim_verify.py", + "wd": "wai", + "body": "set -euo pipefail\nfor seed in 1 2; do\n python3 staged-measure-conformance/sim_diff.py \"$RUNNER_TEMP/wai-bins/union/wai_stage_measure\" \"$seed\" 1500\ndone" + }, + { + "step": "live events · wai-rs and live_events_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_events::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_events)\npython3 -I live-conformance/live_events_verify.py live-conformance/events" + }, + { + "step": "live rooms · wai-rs and live_rooms_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_room::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_room)\npython3 -I live-conformance/live_rooms_verify.py live-conformance/rooms" + }, + { + "step": "capture take · wai-rs and capture_take_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- capture_take:: output_tree::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test capture_take)\npython3 -I live-conformance/capture_take_verify.py live-conformance/takes" + }, + { + "step": "live compose · wai-rs and live_compose_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_compose::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_compose)\npython3 -I live-conformance/live_compose_verify.py live-conformance/compose" + }, + { + "step": "live recording · wai-rs and live_record_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- live_record:: output_tree::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_record)\npython3 -I live-conformance/live_record_verify.py live-conformance/recordings" + }, + { + "step": "timed text · wai-rs and timed_text_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib timed_text::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test timed_text)\npython3 -I live-conformance/timed_text_verify.py live-conformance/captions" + }, + { + "step": "access services · wai-rs and access_services_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib access_services::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test access_services)\npython3 -I live-conformance/access_services_verify.py live-conformance/access" + }, + { + "step": "conformance results · wai-rs and conform_result_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib conform_result::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test conform_result)\n\"$RUNNER_TEMP/wai-bins/union/wai_conform_result_vectors\" verify conform-result-conformance\npython3 -I conform-result-conformance/conform_result_verify.py conform-result-conformance\nfor c in duplicates format members signature summary verdict; do\n python3 -I conform-result-conformance/conform_result_verify.py --skip \"$c\" conform-result-conformance | tail -1\ndone" + }, + { + "step": "live metering · wai-rs and live_metering_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_metering::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_metering)\npython3 -I live-conformance/live_metering_verify.py live-conformance/metering" + }, + { + "step": "energy steering · wai-rs and energy_steering_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib energy_steering::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test energy_steering)\npython3 -I live-conformance/energy_steering_verify.py live-conformance/steering" + }, + { + "step": "sink energy · wai-rs and sink_energy_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib sink_energy::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib sink_meter::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test sink_energy)\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/union/wai_sink_energy_vectors\" verify ../live-conformance/sink-energy)\npython3 -I live-conformance/sink_energy_verify.py live-conformance/sink-energy" + }, + { + "step": "live moderation · wai-rs and live_moderation_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_moderation::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_moderation)\npython3 -I live-conformance/live_moderation_verify.py live-conformance/moderation" + }, + { + "step": "opus deep redundancy · wai-rs and dred_verify.py", + "wd": "wai", + "body": "set -euo pipefail\nd=\"${RUNNER_TEMP:-/tmp}/dred\"\nbash live-conformance/dred/fetch.sh \"$d\" vectors\nw=\"$d/dred_unquantized_weights.bin\"\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib dred_exact::)\n(cd wai-rs && WAI_DRED_WEIGHTS=\"$w\" cargo test --release --no-default-features --features \"$WAI_RELEASE_FEATURES\" --test dred_exact)\n(cd wai-rs && WAI_DRED_WEIGHTS=\"$w\" WAI_DRED_VECTORS=\"$d\" cargo test --release --no-default-features --features \"$WAI_RELEASE_FEATURES\" --test dred_exact_measure)\npython3 -I live-conformance/dred_verify.py live-conformance/dred \"$w\"" + }, + { + "step": "live ads · wai-rs and live_ads_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_ads::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib wai_video_ssai::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_ads)\npython3 -I live-conformance/live_ads_verify.py live-conformance/ads" + }, + { + "step": "key release · wai-rs and key_release_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib key_release::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib wai_video_keys::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test key_release)\npython3 -I live-conformance/key_release_verify.py live-conformance/keys" + }, + { + "step": "ad presentation · wai-rs, ad_presentation_verify.py and the reference producer", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_ads::presentation::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test ad_presentation)\npython3 -I live-conformance/ad_presentation_verify.py live-conformance/presentation\nnode --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/ads/presentation.test.mjs" + }, + { + "step": "stitched presentation · wai-rs, stitched_presentation_verify.py and the reference producer", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib stitched_presentation::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_compose::layout::frame_tests)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test stitched_presentation)\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/union/wai_stitched_presentation\" vectors verify ../live-conformance/stitched)\npython3 -I live-conformance/stitched_presentation_verify.py live-conformance/stitched\nnode --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/ads/stitched.test.mjs" + }, + { + "step": "channel lineup · wai-rs and channel_lineup_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib channel_lineup::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test channel_lineup)\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/union/wai_channel_lineup_vectors\" verify ../live-conformance/channels)\npython3 -I live-conformance/channel_lineup_verify.py live-conformance/channels" + }, + { + "step": "live provenance · wai-rs and c2pa_live_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_provenance::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_provenance)\npython3 -I live-conformance/c2pa_live_verify.py live-conformance/c2pa-live" + }, + { + "step": "synthesis provenance · wai-rs and synthesis_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib synthesis::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test synthesis)\npython3 -I live-conformance/synthesis_verify.py live-conformance/synthesis" + }, + { + "step": "synthesis watermark · wai-rs and synthesis_mark_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- synthesis::mark synthesis_mark::)\n(cd wai-rs && cargo test --release --no-default-features --features \"$WAI_RELEASE_FEATURES\" --test synthesis_mark)\npython3 -I live-conformance/synthesis_mark_verify.py live-conformance/synthesis-mark" + }, + { + "step": "exact upscale · wai-rs and upscale_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib int_sr::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib exact_upscale::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test exact_upscale)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test cli_derive)\npython3 -I live-conformance/upscale_verify.py live-conformance/upscale" + }, + { + "step": "robustness evidence · wai-rs and robustness_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib robustness::)\n(cd wai-rs && cargo test --release --no-default-features --features \"$WAI_RELEASE_FEATURES\" --test robustness)\npython3 -I live-conformance/robustness_verify.py live-conformance/robustness" + }, + { + "step": "prior forms · wai-rs and prior_forms_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- prior_wire:: strict_json:: codecs::prior_form)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test prior_wire)\npython3 -I live-conformance/prior_forms_verify.py live-conformance/prior-forms" + }, + { + "step": "scene carriage · wai-rs and scene_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- scene_map:: strict_json::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test scene_map)\npython3 -I live-conformance/scene_verify.py live-conformance/scene" + }, + { + "step": "faces from keypoints · wai-rs and avatar_face_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib avatar_face::)\n(cd wai-rs && cargo test --release --no-default-features --features \"$WAI_RELEASE_FEATURES\" --test avatar_face)\npython3 -I live-conformance/avatar_face_verify.py live-conformance/avatar-face" + }, + { + "step": "AI usage · wai-rs and ai_usage_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib ai_usage::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib claim_receipt::consumption::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test ai_usage)\npython3 -I live-conformance/ai_usage_verify.py live-conformance/usage\npython3 -I claim-conformance/claim_verify.py consumption live-conformance/usage/vectors.json" + }, + { + "step": "contribution records · wai-rs and contribution_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib contribution::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test contribution)\npython3 -I live-conformance/contribution_verify.py live-conformance/contribution" + }, + { + "step": "live contribution · live_contribution_verify.py", + "wd": "wai", + "body": "set -euo pipefail\npython3 -I live-conformance/live_contribution_verify.py live-conformance/links" + }, + { + "step": "delivery settlement · wai-rs and settlement_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib settlement::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test settlement)\npython3 -I live-conformance/settlement_verify.py live-conformance/settlement" + }, + { + "step": "receipt ledger · wai-rs and receipt_ledger_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib receipt_ledger::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test receipt_ledger)\npython3 -I live-conformance/receipt_ledger_verify.py live-conformance/ledger" + }, + { + "step": "revenue share · wai-rs and revenue_share_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib revenue_share::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib wai_video_channel::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test revenue_share)\npython3 -I live-conformance/revenue_share_verify.py live-conformance/share" + }, + { + "step": "opportunity report · wai-rs and opportunity_report_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib opportunity_report::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test opportunity_report)\npython3 -I live-conformance/opportunity_report_verify.py live-conformance/opportunity" + }, + { + "step": "stream audit · wai-rs and stream_audit_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib stream_audit::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test stream_audit)\npython3 -I live-conformance/stream_audit_verify.py live-conformance/audit" + }, + { + "step": "fleet · wai-rs and fleet_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib fleet::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test fleet)\npython3 -I live-conformance/fleet_verify.py live-conformance/fleet" + }, + { + "step": "route record · wai-rs and route_record_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib route_record::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test route_record)\npython3 -I live-conformance/route_record_verify.py live-conformance/routes" + }, + { + "step": "native sink · wai-rs and native_sink_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib native_sink::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test native_sink)\npython3 -I live-conformance/native_sink_verify.py live-conformance/native-sink" + }, + { + "step": "browser sink · wai-rs, wai-live-web and browser_sink_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib browser_sink::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test browser_sink)\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/union/wai_browser_sink_vectors\" verify ../live-conformance/browser)\n(cd wai-live-web && cargo test --locked --features int_video --lib browser_tests)\npython3 -I live-conformance/browser_sink_verify.py live-conformance/browser" + }, + { + "step": "takedown · wai-rs and takedown_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib takedown::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test takedown)\npython3 -I live-conformance/takedown_verify.py live-conformance/takedown" + }, + { + "step": "influence attribution · wai-rs and influence_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib influence::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test influence)\npython3 -I live-conformance/influence_verify.py live-conformance/influence" + }, + { + "step": "direction records · wai-rs and direction_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib direction::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test direction)\npython3 -I live-conformance/direction_verify.py live-conformance/direction" + }, + { + "step": "co-performance · wai-rs and co_performance_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib co_performance::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test co_performance)\npython3 -I live-conformance/co_performance_verify.py live-conformance/coperformance" + }, + { + "step": "age assurance · wai-rs and age_assurance_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib age_assurance::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test age_assurance)\npython3 -I live-conformance/age_assurance_verify.py live-conformance/age" + }, + { + "step": "bearer receipts · wai-rs and bearer_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib bearer)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test bearer_receipts)\npython3 -I live-conformance/bearer_verify.py live-conformance/bearers\npython3 -I claim-conformance/claim_verify.py bearer live-conformance/bearers/vectors.json" + }, + { + "step": "carousel schedule · wai-rs and carousel_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- carousel:: claim_receipt::schedule::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test carousel)\npython3 -I live-conformance/carousel_verify.py live-conformance/carousel\npython3 -I claim-conformance/claim_verify.py carousel live-conformance/carousel/vectors.json" + }, + { + "step": "holdings · wai-rs and holdings_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- holdings::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test holdings)\npython3 -I live-conformance/holdings_verify.py live-conformance/holdings" + }, + { + "step": "exploration · wai-rs and exploration_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib exploration::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test exploration)\npython3 -I live-conformance/exploration_verify.py live-conformance/exploration" + }, + { + "step": "platform-encode · wai-rs and encode_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --locked --no-default-features --features \"$WAI_FEATURES\" --lib platform_encode::)\n(cd wai-rs && cargo test --locked --no-default-features --features \"$WAI_FEATURES\" --test platform_encode)\npython3 -I encode-conformance/encode_verify.py encode-conformance" + }, + { + "step": "co-watch · wai-rs, co_watch_verify.py and the page's module", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib co_watch::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test co_watch)\npython3 -I live-conformance/co_watch_verify.py live-conformance/cowatch\nnode --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/room/cowatch.test.mjs" + }, + { + "step": "remote desktop · wai-rs, remote_desktop_verify.py and the page's module", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- remote_desktop:: hpke::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test remote_desktop)\npython3 -I live-conformance/remote_desktop_verify.py live-conformance/remote\nnode --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.test.mjs" + }, + { + "step": "QoE aggregation · wai-rs and telemetry_aggregate_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- telemetry_aggregate:: telemetry::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test telemetry_aggregate)\npython3 -I live-conformance/telemetry_aggregate_verify.py live-conformance/aggregate" + }, + { + "step": "exact tone mapping · wai-rs and tonemap_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib -- exact_tonemap:: png_still::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test exact_tonemap)\npython3 -I live-conformance/tonemap_verify.py live-conformance/tonemap" + }, + { + "step": "exact splats · wai-rs and splat_exact_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib splat_exact::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test splat_exact)\npython3 -I live-conformance/splat_exact_verify.py live-conformance/splat" + }, + { + "step": "stream termination · wai-rs, wai-live-web and stream_termination_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib stream_termination::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test stream_termination)\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/union/wai_stream_termination_vectors\" verify ../live-conformance/termination)\n(cd wai-live-web && cargo test --locked --features int_video --lib termination_tests)\npython3 -I live-conformance/stream_termination_verify.py live-conformance/termination" + }, + { + "step": "live worlds · wai-rs and live_worlds_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_world::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_world)\npython3 -I live-conformance/live_worlds_verify.py live-conformance/worlds" + }, + { + "step": "end-to-end encrypted rooms · e2ee_verify.py", + "wd": "wai", + "body": "python3 -I live-conformance/e2ee_verify.py live-conformance/e2ee" + }, + { + "step": "live messaging · wai-rs and live_messaging_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_messaging)\npython3 -I live-conformance/live_messaging_verify.py live-conformance/messaging" + }, + { + "step": "forensic watermark sessions · wai-rs and live_watermark_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib live_watermark::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test live_watermark)\npython3 -I live-conformance/live_watermark_verify.py live-conformance/watermark" + }, + { + "step": "deriving edge · wai-rs and edge_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib derive_edge::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib png_still::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test derive_edge)\npython3 -I edge-conformance/edge_verify.py edge-conformance" + }, + { + "step": "cue injection and Enhanced RTMP · injection_verify.py and ertmp_verify.py", + "wd": "wai", + "body": "set -euo pipefail\npython3 -I live-conformance/injection_verify.py --self-test live-conformance/injection\npython3 -I live-conformance/ertmp_verify.py --self-test live-conformance/ertmp" + }, + { + "step": "quantum referee.py", + "wd": ".", + "body": "python3 -I wai/quantum-conformance/tools/referee.py wai/quantum-conformance" + }, + { + "step": "hybrid signatures · wai-rs and pq_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_PQ_FEATURES\" --lib pq_sig::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_PQ_FEATURES\" --lib synthesis::pq_tests)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_PQ_FEATURES\" --test pq_signatures)\npython3 -I pq-conformance/pq_verify.py pq-conformance\npython3 -I pq-conformance/pq_verify.py verdicts pq-conformance/vectors.json > \"$RUNNER_TEMP/pq-py.tsv\"\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/pq/wai_pq_vectors\" verdicts ../pq-conformance/vectors.json) > \"$RUNNER_TEMP/pq-rs.tsv\"\ndiff \"$RUNNER_TEMP/pq-py.tsv\" \"$RUNNER_TEMP/pq-rs.tsv\"" + }, + { + "step": "claim_verify.py, differential.py, jwp_verify.py, chain_differential.py, chain_gap_differential.py, signer_verify.py", + "wd": "wai", + "body": "set -euo pipefail\npython3 claim-conformance/claim_verify.py claim-conformance/vectors.json\npython3 jwp-conformance/jwp_verify.py jwp-conformance/vectors.json\npython3 signer-conformance/signer_verify.py signer-conformance/vectors.json\npython3 signer-conformance/signer_verify.py verdicts signer-conformance/vectors.json > \"$RUNNER_TEMP/signer-py.tsv\"\n# Each differential against both builds the first step copied (see its comment).\nfor build in union pq; do\n bins=\"$RUNNER_TEMP/wai-bins/$build\"\n if [ \"$build\" = pq ]; then features=$WAI_PQ_FEATURES; else features=$WAI_FEATURES; fi\n echo \"::group::the differentials against the $build build (features $features)\"\n python3 claim-conformance/differential.py \"$bins/wai_claim_vectors\" --random 2000 --seed 1\n for seed in 1 2; do\n python3 jwp-conformance/chain_differential.py \"$bins/wai_jwp_vectors\" --n 3000 --seed \"$seed\"\n python3 jwp-conformance/chain_gap_differential.py \"$bins/wai_jwp_vectors\" --n 400 --seed \"$seed\"\n done\n \"$bins/wai_signer_vectors\" verdicts signer-conformance/vectors.json > \"$RUNNER_TEMP/signer-rs-$build.tsv\"\n diff \"$RUNNER_TEMP/signer-py.tsv\" \"$RUNNER_TEMP/signer-rs-$build.tsv\"\n echo \"the $build build: every differential agrees\"\n echo \"::endgroup::\"\ndone" + }, + { + "step": "header probes · wai-rs, the fetched LCEVC streams and format_probe_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib format_probe::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test format_probe)\nfetched=$(mktemp -d)\nbash format-probe-conformance/fetch.sh \"$fetched\"\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/union/wai_format_probe\" verify ../format-probe-conformance --fetched \"$fetched\")\npython3 -I format-probe-conformance/format_probe_verify.py format-probe-conformance --fetched \"$fetched\"" + }, + { + "step": "media identification · wai-rs and identify_verify.py", + "wd": "wai", + "body": "set -euo pipefail\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --lib identify::)\n(cd wai-rs && cargo test --no-default-features --features \"$WAI_FEATURES\" --test identify)\n(cd wai-rs && \"$RUNNER_TEMP/wai-bins/union/wai_identify_vectors\" verify ../identify-conformance)\npython3 -I identify-conformance/identify_verify.py identify-conformance" + }, + { + "step": "cmsd_read_verify.py", + "wd": "wai", + "body": "python3 -I telemetry-conformance/cmsd_read_verify.py telemetry-conformance" + }, + { + "step": "loc_verify.py", + "wd": "wai", + "body": "python3 -I live-conformance/loc_verify.py live-conformance/loc" + } + ], + "int_kernels_debug_features": [ + "audio_int", + "generate", + "i32_forms", + "legacy_int", + "neural_int", + "score", + "splat_int", + "stage_measure", + "staged", + "video_int", + "world_int" + ], + "int_kernels_debug_tests": [ + "cli_generate_score", + "cli_stage", + "generate", + "int_audio", + "int_audio_ops", + "int_hyper_ops", + "int_kernels_geometry", + "int_kernels_vectors", + "int_mlicpp", + "int_mlicpp_ops", + "int_mlicpp_preflight", + "int_ops_conformance", + "int_ops_differential", + "int_payload", + "int_payload_cost", + "int_payload_preflight", + "int_set_cache", + "int_splat", + "int_splat_ops", + "int_synth_ops", + "int_video", + "int_video_enc_memory", + "int_video_halfpel", + "int_video_keyframe", + "int_video_ops", + "int_world", + "int_world2", + "int_world_memory", + "int_world_ops", + "jpegai_bitstream_calibration", + "jpegai_nets_ops", + "jpegai_ops", + "stage_measure", + "stage_measure_bounds", + "staged_resources", + "wir1_ops" + ], + "int_kernels_release_features": [ + "audio_int", + "generate", + "i32_forms", + "legacy_int", + "neural_int", + "score", + "splat_int", + "stage_measure", + "staged", + "video_int", + "world_int" + ], + "int_kernels_release_tests": [ + "cli_generate_score", + "cli_stage", + "generate", + "int_audio", + "int_audio_ops", + "int_hyper_ops", + "int_kernels_geometry", + "int_kernels_vectors", + "int_mlicpp", + "int_mlicpp_ops", + "int_mlicpp_preflight", + "int_ops_conformance", + "int_ops_differential", + "int_payload", + "int_payload_cost", + "int_payload_preflight", + "int_set_cache", + "int_splat", + "int_splat_ops", + "int_synth_ops", + "int_video", + "int_video_enc_memory", + "int_video_halfpel", + "int_video_keyframe", + "int_video_ops", + "int_world", + "int_world2", + "int_world_memory", + "int_world_ops", + "jpegai_bitstream_calibration", + "jpegai_nets_ops", + "jpegai_ops", + "manifest_parse_memory", + "manifest_reader_equivalence", + "stage_measure", + "stage_measure_bounds", + "staged_resources", + "wir1_ops" + ], + "int_kernels_amd_features": [ + "audio_int", + "generate", + "i32_forms", + "legacy_int", + "neural_int", + "score", + "splat_int", + "stage_measure", + "staged", + "video_int", + "world_int" + ], + "int_kernels_amd_tests": [ + "int_audio_ops", + "int_hyper_ops", + "int_kernels_geometry", + "int_kernels_vectors", + "int_mlicpp_ops", + "int_ops_conformance", + "int_splat_ops", + "int_synth_ops", + "int_video_ops", + "int_world_ops", + "jpegai_nets_ops", + "jpegai_ops", + "wir1_ops" + ], + "corpora_features": [ + "audio_int", + "audio_scene", + "film", + "haptic", + "http", + "key_discovery", + "neural_int", + "oer", + "pq_signatures", + "provenance", + "quantum", + "quantum_ops", + "receipt_claims", + "score", + "splat4d", + "splat_int", + "stage_measure", + "staged", + "telemetry", + "video_int", + "video_manifest", + "world" + ], + "corpora": [ + "worlds wai_world_vectors ../world-conformance", + "spatial-audio wai_audio_vectors ../audio-conformance", + "haptics wai_haptic_vectors ../haptic-conformance", + "state-feeds wai_feed_vectors ../feed-conformance", + "avatars wai_avatar_vectors ../avatar-conformance", + "volumetric wai_splat4d_vectors ../volumetric-conformance", + "score wai_score_vectors ../score-conformance", + "int-synth wai_int_synth_vectors ../neural-conformance", + "film wai_film_vectors ../film-conformance", + "quantum wai_quantum_vectors ../quantum-conformance", + "quantum-ops wai_quantum_ops_vectors ../quantum-ops-conformance", + "oer wai_oer_vectors ../oer-conformance", + "jwp wai_jwp_vectors ../jwp-conformance", + "claims wai_claim_vectors ../claim-conformance", + "pin wai_pin_vectors ../pin-conformance", + "signers wai_signer_vectors ../signer-conformance", + "int-payload wai_int_payload_vectors ../int-payload-conformance", + "staged wai_staged_vectors ../staged-conformance", + "http wai_http_vectors ../http-conformance", + "staged-measure wai_stage_measure ../staged-measure-conformance", + "telemetry wai_telemetry ../telemetry-conformance", + "pq-signatures wai_pq_vectors ../pq-conformance" + ] +} diff --git a/.github/wai-conformance/corpora/avatars.txt b/.github/wai-conformance/corpora/avatars.txt new file mode 100644 index 000000000..336f42f24 --- /dev/null +++ b/.github/wai-conformance/corpora/avatars.txt @@ -0,0 +1,4 @@ +# corpus: avatars +# order: 50 +# bin: wai_avatar_vectors +# dir: ../avatar-conformance diff --git a/.github/wai-conformance/corpora/claims.txt b/.github/wai-conformance/corpora/claims.txt new file mode 100644 index 000000000..6be303bb5 --- /dev/null +++ b/.github/wai-conformance/corpora/claims.txt @@ -0,0 +1,4 @@ +# corpus: claims +# order: 140 +# bin: wai_claim_vectors +# dir: ../claim-conformance diff --git a/.github/wai-conformance/corpora/film.txt b/.github/wai-conformance/corpora/film.txt new file mode 100644 index 000000000..fad5a53ef --- /dev/null +++ b/.github/wai-conformance/corpora/film.txt @@ -0,0 +1,4 @@ +# corpus: film +# order: 90 +# bin: wai_film_vectors +# dir: ../film-conformance diff --git a/.github/wai-conformance/corpora/haptics.txt b/.github/wai-conformance/corpora/haptics.txt new file mode 100644 index 000000000..8306f780d --- /dev/null +++ b/.github/wai-conformance/corpora/haptics.txt @@ -0,0 +1,4 @@ +# corpus: haptics +# order: 30 +# bin: wai_haptic_vectors +# dir: ../haptic-conformance diff --git a/.github/wai-conformance/corpora/http.txt b/.github/wai-conformance/corpora/http.txt new file mode 100644 index 000000000..3a09e59b0 --- /dev/null +++ b/.github/wai-conformance/corpora/http.txt @@ -0,0 +1,4 @@ +# corpus: http +# order: 190 +# bin: wai_http_vectors +# dir: ../http-conformance diff --git a/.github/wai-conformance/corpora/int-payload.txt b/.github/wai-conformance/corpora/int-payload.txt new file mode 100644 index 000000000..3e2dbf67f --- /dev/null +++ b/.github/wai-conformance/corpora/int-payload.txt @@ -0,0 +1,4 @@ +# corpus: int-payload +# order: 170 +# bin: wai_int_payload_vectors +# dir: ../int-payload-conformance diff --git a/.github/wai-conformance/corpora/int-synth.txt b/.github/wai-conformance/corpora/int-synth.txt new file mode 100644 index 000000000..38ba07307 --- /dev/null +++ b/.github/wai-conformance/corpora/int-synth.txt @@ -0,0 +1,4 @@ +# corpus: int-synth +# order: 80 +# bin: wai_int_synth_vectors +# dir: ../neural-conformance diff --git a/.github/wai-conformance/corpora/jwp.txt b/.github/wai-conformance/corpora/jwp.txt new file mode 100644 index 000000000..e56c187f2 --- /dev/null +++ b/.github/wai-conformance/corpora/jwp.txt @@ -0,0 +1,4 @@ +# corpus: jwp +# order: 130 +# bin: wai_jwp_vectors +# dir: ../jwp-conformance diff --git a/.github/wai-conformance/corpora/oer.txt b/.github/wai-conformance/corpora/oer.txt new file mode 100644 index 000000000..004b0aa71 --- /dev/null +++ b/.github/wai-conformance/corpora/oer.txt @@ -0,0 +1,4 @@ +# corpus: oer +# order: 120 +# bin: wai_oer_vectors +# dir: ../oer-conformance diff --git a/.github/wai-conformance/corpora/pin.txt b/.github/wai-conformance/corpora/pin.txt new file mode 100644 index 000000000..28043cb90 --- /dev/null +++ b/.github/wai-conformance/corpora/pin.txt @@ -0,0 +1,4 @@ +# corpus: pin +# order: 150 +# bin: wai_pin_vectors +# dir: ../pin-conformance diff --git a/.github/wai-conformance/corpora/pq-signatures.txt b/.github/wai-conformance/corpora/pq-signatures.txt new file mode 100644 index 000000000..3c82cd002 --- /dev/null +++ b/.github/wai-conformance/corpora/pq-signatures.txt @@ -0,0 +1,4 @@ +# corpus: pq-signatures +# order: 220 +# bin: wai_pq_vectors +# dir: ../pq-conformance diff --git a/.github/wai-conformance/corpora/quantum-ops.txt b/.github/wai-conformance/corpora/quantum-ops.txt new file mode 100644 index 000000000..6a0c8f91a --- /dev/null +++ b/.github/wai-conformance/corpora/quantum-ops.txt @@ -0,0 +1,4 @@ +# corpus: quantum-ops +# order: 110 +# bin: wai_quantum_ops_vectors +# dir: ../quantum-ops-conformance diff --git a/.github/wai-conformance/corpora/quantum.txt b/.github/wai-conformance/corpora/quantum.txt new file mode 100644 index 000000000..a24a8f492 --- /dev/null +++ b/.github/wai-conformance/corpora/quantum.txt @@ -0,0 +1,4 @@ +# corpus: quantum +# order: 100 +# bin: wai_quantum_vectors +# dir: ../quantum-conformance diff --git a/.github/wai-conformance/corpora/score.txt b/.github/wai-conformance/corpora/score.txt new file mode 100644 index 000000000..4e7c7949a --- /dev/null +++ b/.github/wai-conformance/corpora/score.txt @@ -0,0 +1,4 @@ +# corpus: score +# order: 70 +# bin: wai_score_vectors +# dir: ../score-conformance diff --git a/.github/wai-conformance/corpora/signers.txt b/.github/wai-conformance/corpora/signers.txt new file mode 100644 index 000000000..a2ce7b666 --- /dev/null +++ b/.github/wai-conformance/corpora/signers.txt @@ -0,0 +1,4 @@ +# corpus: signers +# order: 160 +# bin: wai_signer_vectors +# dir: ../signer-conformance diff --git a/.github/wai-conformance/corpora/spatial-audio.txt b/.github/wai-conformance/corpora/spatial-audio.txt new file mode 100644 index 000000000..d6531e743 --- /dev/null +++ b/.github/wai-conformance/corpora/spatial-audio.txt @@ -0,0 +1,4 @@ +# corpus: spatial-audio +# order: 20 +# bin: wai_audio_vectors +# dir: ../audio-conformance diff --git a/.github/wai-conformance/corpora/staged-measure.txt b/.github/wai-conformance/corpora/staged-measure.txt new file mode 100644 index 000000000..47ec764e5 --- /dev/null +++ b/.github/wai-conformance/corpora/staged-measure.txt @@ -0,0 +1,4 @@ +# corpus: staged-measure +# order: 200 +# bin: wai_stage_measure +# dir: ../staged-measure-conformance diff --git a/.github/wai-conformance/corpora/staged.txt b/.github/wai-conformance/corpora/staged.txt new file mode 100644 index 000000000..42a082d2f --- /dev/null +++ b/.github/wai-conformance/corpora/staged.txt @@ -0,0 +1,4 @@ +# corpus: staged +# order: 180 +# bin: wai_staged_vectors +# dir: ../staged-conformance diff --git a/.github/wai-conformance/corpora/state-feeds.txt b/.github/wai-conformance/corpora/state-feeds.txt new file mode 100644 index 000000000..830378df4 --- /dev/null +++ b/.github/wai-conformance/corpora/state-feeds.txt @@ -0,0 +1,4 @@ +# corpus: state-feeds +# order: 40 +# bin: wai_feed_vectors +# dir: ../feed-conformance diff --git a/.github/wai-conformance/corpora/telemetry.txt b/.github/wai-conformance/corpora/telemetry.txt new file mode 100644 index 000000000..93bcd0d34 --- /dev/null +++ b/.github/wai-conformance/corpora/telemetry.txt @@ -0,0 +1,4 @@ +# corpus: telemetry +# order: 210 +# bin: wai_telemetry +# dir: ../telemetry-conformance diff --git a/.github/wai-conformance/corpora/volumetric.txt b/.github/wai-conformance/corpora/volumetric.txt new file mode 100644 index 000000000..06a944a8b --- /dev/null +++ b/.github/wai-conformance/corpora/volumetric.txt @@ -0,0 +1,4 @@ +# corpus: volumetric +# order: 60 +# bin: wai_splat4d_vectors +# dir: ../volumetric-conformance diff --git a/.github/wai-conformance/corpora/worlds.txt b/.github/wai-conformance/corpora/worlds.txt new file mode 100644 index 000000000..1be6231b6 --- /dev/null +++ b/.github/wai-conformance/corpora/worlds.txt @@ -0,0 +1,4 @@ +# corpus: worlds +# order: 10 +# bin: wai_world_vectors +# dir: ../world-conformance diff --git a/.github/wai-conformance/verifiers/access-services.sh b/.github/wai-conformance/verifiers/access-services.sh new file mode 100644 index 000000000..8d883103a --- /dev/null +++ b/.github/wai-conformance/verifiers/access-services.sh @@ -0,0 +1,25 @@ +# step: access services · wai-rs and access_services_verify.py +# order: 160 +# features: access_services +# paths: wai/extensions/access-services.md +# +# wai.access.services (extensions/access-services.md): the reference module's unit tests +# (words, full and prefix alignment with their walk back, signed integers, gains) and +# tests/access_services.rs (vectors.json and each DAPT document regenerate byte for byte +# from their fixed seeds and no other file is there, wai-rs gives every vector and +# document its stated outcome and figures, every reason of §10 refuses a case, and §10 +# lists exactly the reference's codes); then the corpus's second implementation, +# access_services_verify.py, written from the extension's text and sharing no code with +# wai-rs, which verifies events and captions with timed_text_verify.py and every record +# with WAI's strict Ed25519 rule, computes every figure and writes every DAPT document +# itself, parses each with xml.etree, reads the extension's reason table, and requires +# every mutated copy of the corpus to fail; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/access-services.md: tests/access_services.rs requires this extension's §10 to +# list the codes the reference reports, access_services_verify.py reads that table, and the +# registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib access_services::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test access_services) +python3 -I live-conformance/access_services_verify.py live-conformance/access diff --git a/.github/wai-conformance/verifiers/ad-presentation.sh b/.github/wai-conformance/verifiers/ad-presentation.sh new file mode 100644 index 000000000..548f1e8cb --- /dev/null +++ b/.github/wai-conformance/verifiers/ad-presentation.sh @@ -0,0 +1,28 @@ +# step: ad presentation · wai-rs, ad_presentation_verify.py and the reference producer +# order: 250 +# features: live_ads +# paths: wai/extensions/ad-presentation.md +# +# wai.live.ad_presentation (extensions/ad-presentation.md): the reference module's unit +# tests (claims round-trip and their rules refuse, the quartile, the qualifying run, the +# touched objects, a policy's presentation fields, RFC 9577's TokenChallenge) and +# tests/ad_presentation.rs (every file of the committed corpus regenerates byte for byte +# from its fixed seeds and no other file is there, RFC 9578's own token verifies under its +# key, wai-rs gives every room its outcome and every claim its verdict, and §11 lists +# exactly the reference's codes); then the corpus's second implementation, +# ad_presentation_verify.py, written from the extension's text and sharing no code with +# wai-rs, which verifies each room with live_ads_verify.py, judges every claim and its +# RSASSA-PSS itself, requires every mutated copy of the corpus to fail and catches each of +# its own checks dropped; standard library only, run isolated (-I). Last, the browser +# client's reference producer (joule-moq-web/web/ads), which records observation +# sequences into spans and writes and signs the corpus's claims from them, byte for byte. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/ad-presentation.md: tests/ad_presentation.rs requires this extension's §11 to +# list the codes the reference reports, ad_presentation_verify.py reads that table, and the +# registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_ads::presentation::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test ad_presentation) +python3 -I live-conformance/ad_presentation_verify.py live-conformance/presentation +node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/ads/presentation.test.mjs diff --git a/.github/wai-conformance/verifiers/age-assurance.sh b/.github/wai-conformance/verifiers/age-assurance.sh new file mode 100644 index 000000000..217b1b7df --- /dev/null +++ b/.github/wai-conformance/verifiers/age-assurance.sh @@ -0,0 +1,27 @@ +# step: age assurance · wai-rs and age_assurance_verify.py +# order: 520 +# features: age_assurance +# paths: wai/extensions/age-assurance.md +# +# wai.age.assurance (extensions/age-assurance.md): the reference module's unit tests (the +# pinned AlgorithmIdentifier and RFC 9578's first vector through this module, the +# TokenChallenge read exactly, the suppression floor, the grant attribute `age`, a span's +# coverage) and tests/age_assurance.rs (vectors.json regenerates byte for byte from its +# fixed seeds and no other file is there, wai-rs gives every case its stated outcome, and +# §11 lists exactly the reference's codes, each of which the corpus gives); then the +# corpus's second implementation, age_assurance_verify.py, written from the extension's +# text and sharing no code with wai-rs, which reads every record, writes JCS and verifies +# every token's RSASSA-PSS itself, reads rooms with live_rooms_verify.py, checks signatures +# with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, reads the extension's +# reason table, requires every mutated copy of the corpus to fail, and drops each of its +# own checks in turn and requires the corpus to catch the weakened verifier; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/age-assurance.md: tests/age_assurance.rs requires this extension's §11 to list +# the codes the reference reports, age_assurance_verify.py reads that table, and the registry's +# wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib age_assurance::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test age_assurance) +python3 -I live-conformance/age_assurance_verify.py live-conformance/age diff --git a/.github/wai-conformance/verifiers/ai-usage.sh b/.github/wai-conformance/verifiers/ai-usage.sh new file mode 100644 index 000000000..59a9645c1 --- /dev/null +++ b/.github/wai-conformance/verifiers/ai-usage.sh @@ -0,0 +1,32 @@ +# step: AI usage · wai-rs and ai_usage_verify.py +# order: 360 +# features: ai_usage +# paths: wai/extensions/ai-usage.md +# +# wai.ai.usage (extensions/ai-usage.md): the reference module's unit tests (RFC 9651's +# Dictionary parser against the vocabulary's examples and the RFC's edges, statements +# read and refused in their steps' order, the precedence rule, HTTP fields compared by +# meaning), the consumption class's body and terms in claim_receipt, and +# tests/ai_usage.rs (vectors.json regenerates byte for byte from its fixed seeds and no +# other file is there, wai-rs gives every statement, object, room, response, catalog, +# claim and gate its stated outcome, §9 lists exactly the reference's codes, each of which +# the corpus gives, and jwp-receipts §3.1 lists the class rules); then the corpus's second +# implementation, ai_usage_verify.py, written from the extension's text and sharing no +# code with wai-rs, which parses Structured Fields, canonical forms, receipts and claims +# itself, reads rooms with live_rooms_verify.py, checks signatures with WAI's strict +# Ed25519 rule and BLAKE3 from claim-conformance, reads the extension's reason table, +# requires every mutated copy of the corpus to fail, and drops each of its own checks in +# turn and requires the corpus to catch the weakened verifier; standard library only, run +# isolated (-I). Last, claim-conformance's own verifier reads every consumption claim of +# the corpus through jwp-receipts §3.1 steps 1 to 4 and requires the stated step and reason. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/ai-usage.md: tests/ai_usage.rs requires this extension's §9 to list the codes the +# reference reports, ai_usage_verify.py reads that table, and the registry's wire identifiers +# test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib ai_usage::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib claim_receipt::consumption::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test ai_usage) +python3 -I live-conformance/ai_usage_verify.py live-conformance/usage +python3 -I claim-conformance/claim_verify.py consumption live-conformance/usage/vectors.json diff --git a/.github/wai-conformance/verifiers/avatar-face.sh b/.github/wai-conformance/verifiers/avatar-face.sh new file mode 100644 index 000000000..dbd0fa33f --- /dev/null +++ b/.github/wai-conformance/verifiers/avatar-face.sh @@ -0,0 +1,28 @@ +# step: faces from keypoints · wai-rs and avatar_face_verify.py +# order: 350 +# features: avatar_face +# release-features: avatar_face +# paths: wai/extensions/avatars.md +# +# wai.avatar.face (extensions/avatars.md §6 to §13): the module's unit tests (the +# generator's wai-ops/1 graph against §7.6's arithmetic written out loop by loop; the warp +# at rest is the reference; the mask's table; the files read and refused in their order), +# and tests/avatar_face.rs (the committed corpus regenerates byte for byte from its fixed +# seeds and its pinned inputs, and no other file is there; wai-rs gives every form, frame +# and case what it states; §10 lists exactly the reference's codes; §7.9's figures are +# training.json's; the registered generator meets acc64 and not acc32); then the corpus's +# second implementation, avatar_face_verify.py, written from the extension's text and +# sharing no code with wai-rs, which runs the driver and the generator's arithmetic in +# Python's integers, verifies the claims, the log and the label policy with +# synthesis_verify.py, and requires every mutated copy of the corpus to fail; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/avatars.md: tests/avatar_face.rs requires this extension's §10 to list the codes +# the reference reports and its §7.9 to state the training report's figures, +# avatar_face_verify.py reads §10, and the registry's tests read SPEC §5's registration of +# wai.avatar.face. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib avatar_face::) +(cd wai-rs && cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test avatar_face) +python3 -I live-conformance/avatar_face_verify.py live-conformance/avatar-face diff --git a/.github/wai-conformance/verifiers/bearer-receipts.sh b/.github/wai-conformance/verifiers/bearer-receipts.sh new file mode 100644 index 000000000..fe7f1c3c9 --- /dev/null +++ b/.github/wai-conformance/verifiers/bearer-receipts.sh @@ -0,0 +1,31 @@ +# step: bearer receipts · wai-rs and bearer_verify.py +# order: 530 +# features: bearer_receipts +# paths: wai/extensions/bearer-receipts.md +# +# wai.bearer.receipts (extensions/bearer-receipts.md): the reference module's unit tests +# (the bound, the bitmap rules, the bearer classes, and the corpus reaching every stated +# outcome and failing a changed one) and tests/bearer_receipts.rs (vectors.json +# regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives +# every case its stated outcome, the cases that carry one receipt on every bearer class +# carry the same bytes with the same object verdicts, §10 lists exactly the reference's +# codes, each of which the corpus gives, and jwp-receipts §3.1 lists the class codes at +# their steps); then the corpus's second implementation, bearer_verify.py, written from +# the extension's text and sharing no code with wai-rs, which verifies groups and objects +# with jwp_verify.py, claims through jwp-receipts §3.1 steps 1-5 with claim_verify.py and +# confirmations with signer_verify.py (each needs the `cryptography` package installed +# above), writes the context checks, the verdict, the bound and offline confirmation +# itself, reads the extension's reason table, requires every mutated copy of the corpus +# to fail, and drops each of its own checks in turn and requires the corpus to catch the +# weakened verifier; run isolated (-I). Last, claim-conformance's own verifier reads every +# transmission and reconciliation claim of the corpus through jwp-receipts §3.1 steps 1 +# to 4 and requires the stated step and reason. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/bearer-receipts.md: tests/bearer_receipts.rs requires this extension's §10 to +# list the codes the reference reports, and bearer_verify.py reads that table. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib bearer) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test bearer_receipts) +python3 -I live-conformance/bearer_verify.py live-conformance/bearers +python3 -I claim-conformance/claim_verify.py bearer live-conformance/bearers/vectors.json diff --git a/.github/wai-conformance/verifiers/browser-sink.sh b/.github/wai-conformance/verifiers/browser-sink.sh new file mode 100644 index 000000000..6a4806112 --- /dev/null +++ b/.github/wai-conformance/verifiers/browser-sink.sh @@ -0,0 +1,34 @@ +# step: browser sink · wai-rs, wai-live-web and browser_sink_verify.py +# order: 470 +# features: browser_sink browser_sink,receipt_claims +# paths: wai/extensions/browser-sink.md +# +# The browser sink (extensions/browser-sink.md): the reference module's unit tests (the +# registry gate's rows, registered tiers, integers as §5 reads them, a binding's +# canonical form, an answer for an unregistered codec string never read, powerEfficient +# never stated off the user agent's route, the side table's ambiguity) and +# tests/browser_sink.rs (both files of the committed corpus regenerate byte for byte and +# no other file is there, wai-rs gives every case its stated outcome, every reason of +# §11 appears in the corpus and §11 lists exactly the reference's codes, and the chain +# the generator signs by hand verifies under jwp_delivery); wai-live-web's browser +# functions held to the same corpus natively, and its wasm-route decoder verifying, +# decoding and binding every chain object to the published frame digests; then the +# corpus's second implementation, browser_sink_verify.py, written from the extension's +# text and sharing no code with wai-rs, which reads the registry export for each +# capability's class, recomputes every route, evaluation, energy entry, binding, chain +# signature, frame state and side table, changes the answers for unregistered codec +# strings every way and requires the corpus still to pass, and requires every mutated +# copy of the corpus to fail; standard library only, run isolated (-I). The browser's +# own run of it is joule-moq-web.yml's scenario binding. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/browser-sink.md: tests/browser_sink.rs requires this extension's §11 to list the +# codes the reference reports, browser_sink_verify.py reads its §2, §4.1, §4.2 and §11 tables +# and capability-constraints' §3.1 register, and the registry's wire identifiers test reads its +# sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib browser_sink::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test browser_sink) +(cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_browser_sink_vectors" verify ../live-conformance/browser) +(cd wai-live-web && cargo test --locked --features int_video --lib browser_tests) +python3 -I live-conformance/browser_sink_verify.py live-conformance/browser diff --git a/.github/wai-conformance/verifiers/capture-take.sh b/.github/wai-conformance/verifiers/capture-take.sh new file mode 100644 index 000000000..9e2e70abb --- /dev/null +++ b/.github/wai-conformance/verifiers/capture-take.sh @@ -0,0 +1,23 @@ +# step: capture take · wai-rs and capture_take_verify.py +# order: 120 +# features: capture_take +# paths: wai/extensions/capture-binding.md +# +# wai.capture.take (extensions/capture-binding.md): the reference module's unit tests, +# the output tree's (whose RFC 6962 hash and RFC 9162 consistency proofs the take tree +# reuses), and tests/capture_take.rs (the committed corpus regenerates byte for byte +# from its fixed seeds, wai-rs gives every vector its stated outcome, every reason of +# §12 appears in it, and §12 lists exactly the reference's codes); then the corpus's +# second implementation, capture_take_verify.py, written from the extension's text and +# sharing no code with wai-rs, which verifies each room with live_rooms_verify.py and +# every signature with WAI's strict Ed25519 (claim-conformance/_ed25519.py), and +# requires every mutated copy of the corpus to fail; standard library only, run +# isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/capture-binding.md: tests/capture_take.rs requires this extension's §12 to list +# the codes the reference reports, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- capture_take:: output_tree::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test capture_take) +python3 -I live-conformance/capture_take_verify.py live-conformance/takes diff --git a/.github/wai-conformance/verifiers/carousel.sh b/.github/wai-conformance/verifiers/carousel.sh new file mode 100644 index 000000000..075cec977 --- /dev/null +++ b/.github/wai-conformance/verifiers/carousel.sh @@ -0,0 +1,29 @@ +# step: carousel schedule · wai-rs and carousel_verify.py +# order: 540 +# features: carousel +# paths: wai/extensions/carousel.md +# +# wai.bearer.carousel (extensions/carousel.md): the reference module's unit tests (each code's +# bounds and division into source blocks, K', the passes of advance and repeat, plans, and the +# corpus reaching every stated outcome and failing a changed one) and tests/carousel.rs +# (vectors.json regenerates byte for byte from its fixed seeds, wai-rs gives every case its +# stated outcome, §9 lists exactly the reference's codes, Appendix A is its table of K', +# jwp-receipts §3.1 lists the class codes at their steps, K' and Partition[] agree with an +# independent RFC 6330 implementation for every block size, and each RaptorQ object a case +# holds available decodes with that implementation from what a receiver within the case's +# loss bound holds); then the corpus's second implementation, carousel_verify.py, written +# from the extension's text and sharing no code with wai-rs, which verifies groups and +# objects with jwp_verify.py and claims through steps 1-5 with claim_verify.py, writes the +# context checks, the availability and the plans itself, reads §9 and Appendix A, requires +# every mutated copy of the corpus to fail, and drops each of its own checks in turn; run +# isolated (-I). Last, claim-conformance's own verifier reads every claim of the corpus +# through jwp-receipts §3.1 steps 1 to 4 and requires the stated outcome. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/carousel.md: tests/carousel.rs requires §9 to list the reference's codes and +# Appendix A to be its table of K', and carousel_verify.py reads both. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- carousel:: claim_receipt::schedule::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test carousel) +python3 -I live-conformance/carousel_verify.py live-conformance/carousel +python3 -I claim-conformance/claim_verify.py carousel live-conformance/carousel/vectors.json diff --git a/.github/wai-conformance/verifiers/catalogue-prior.sh b/.github/wai-conformance/verifiers/catalogue-prior.sh new file mode 100644 index 000000000..baf7b855e --- /dev/null +++ b/.github/wai-conformance/verifiers/catalogue-prior.sh @@ -0,0 +1,11 @@ +# step: catalogue prior · the record and the text quoting it +# order: 30 +# +# The catalogue-prior record (research/catalogue-prior.md): each reading checked against +# the SHA-256 its run.json states; every BD-rate the run stored recomputed from the +# rate and quality points, and the rate proxy, the recurrence and the decision rule's +# verdicts from the readings, by a second implementation that shares no code with the +# scripts that took them (tools/wai_catalogue_prior_verify.py, standard library only, +# run isolated); and every figure the page quotes checked (each is followed by a `cat` +# marker naming its value). +python3 -I tools/wai_catalogue_prior_verify.py research/catalogue-prior research/catalogue-prior.md diff --git a/.github/wai-conformance/verifiers/channel-lineup.sh b/.github/wai-conformance/verifiers/channel-lineup.sh new file mode 100644 index 000000000..707bf318d --- /dev/null +++ b/.github/wai-conformance/verifiers/channel-lineup.sh @@ -0,0 +1,26 @@ +# step: channel lineup · wai-rs and channel_lineup_verify.py +# order: 270 +# features: channel_lineup +# paths: wai/extensions/channel-lineup.md +# +# wai.video.lineup (extensions/channel-lineup.md): the reference module's unit tests (times, +# the document type declaration, the reasons) and tests/channel_lineup.rs (every file of the +# committed corpus regenerates byte for byte from its fixed seeds, and no other file is +# there; wai-rs gives every import and every verification its outcome; §10 lists exactly +# the reference's codes; §9's commands, run as the text writes them through `wai`, yield +# the corpus's receipt, lineup, cues and guides, and a refusal is its reason with exit 1); +# then the corpus's second implementation, channel_lineup_verify.py, written from the +# extension's text and sharing no code with wai-rs, which reimports every vector, requires +# every outcome and fact, and requires every mutated copy of the corpus to fail and every +# check it drops to be caught. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/channel-lineup.md: tests/channel_lineup.rs requires this extension's §10 to list +# the codes the reference adds and runs its §9's commands as the text writes them, +# channel_lineup_verify.py reads its §10, and the registry's wire identifiers test reads its +# sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib channel_lineup::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test channel_lineup) +(cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_channel_lineup_vectors" verify ../live-conformance/channels) +python3 -I live-conformance/channel_lineup_verify.py live-conformance/channels diff --git a/.github/wai-conformance/verifiers/claims-jwp-signers.sh b/.github/wai-conformance/verifiers/claims-jwp-signers.sh new file mode 100644 index 000000000..9216879ea --- /dev/null +++ b/.github/wai-conformance/verifiers/claims-jwp-signers.sh @@ -0,0 +1,32 @@ +# step: claim_verify.py, differential.py, jwp_verify.py, chain_differential.py, chain_gap_differential.py, signer_verify.py +# order: 720 +# features: key_discovery +# bins: wai_claim_vectors wai_jwp_vectors wai_signer_vectors +# pq-bins: wai_claim_vectors wai_jwp_vectors wai_signer_vectors +# +# chain_differential.py signs receipt chains around jwp-receipts §3 step 3 and §6.1 +# (two keys, two tracks, both payloads, replays, forks, gaps, the guard and the +# freshness bound), and chain_gap_differential.py genuine chains up to 90 groups +# delivered shuffled, withheld and walked back with rewritten links and forks; both +# require wai_jwp_vectors and jwp_verify.py to give every chain the same verdict. Two +# fixed seeds each, so a run is reproducible. +set -euo pipefail +python3 claim-conformance/claim_verify.py claim-conformance/vectors.json +python3 jwp-conformance/jwp_verify.py jwp-conformance/vectors.json +python3 signer-conformance/signer_verify.py signer-conformance/vectors.json +python3 signer-conformance/signer_verify.py verdicts signer-conformance/vectors.json > "$RUNNER_TEMP/signer-py.tsv" +# Each differential against both builds the first step copied (see its comment). +for build in union pq; do + bins="$RUNNER_TEMP/wai-bins/$build" + if [ "$build" = pq ]; then features=$WAI_PQ_FEATURES; else features=$WAI_FEATURES; fi + echo "::group::the differentials against the $build build (features $features)" + python3 claim-conformance/differential.py "$bins/wai_claim_vectors" --random 2000 --seed 1 + for seed in 1 2; do + python3 jwp-conformance/chain_differential.py "$bins/wai_jwp_vectors" --n 3000 --seed "$seed" + python3 jwp-conformance/chain_gap_differential.py "$bins/wai_jwp_vectors" --n 400 --seed "$seed" + done + "$bins/wai_signer_vectors" verdicts signer-conformance/vectors.json > "$RUNNER_TEMP/signer-rs-$build.tsv" + diff "$RUNNER_TEMP/signer-py.tsv" "$RUNNER_TEMP/signer-rs-$build.tsv" + echo "the $build build: every differential agrees" + echo "::endgroup::" +done diff --git a/.github/wai-conformance/verifiers/cmsd-read.sh b/.github/wai-conformance/verifiers/cmsd-read.sh new file mode 100644 index 000000000..dab58d4c6 --- /dev/null +++ b/.github/wai-conformance/verifiers/cmsd-read.sh @@ -0,0 +1,8 @@ +# step: cmsd_read_verify.py +# order: 750 +# +# CMSD reading (extensions/delivery-telemetry.md §6.6): the second implementation, written +# from that text and RFC 9651 with its own field parser, reads every field of +# telemetry-conformance/cmsd-read/ and every field expected/cmsd.json states, and requires +# every mutated copy to fail. wai-rs's reading is checked by wai_telemetry verify above. +python3 -I telemetry-conformance/cmsd_read_verify.py telemetry-conformance diff --git a/.github/wai-conformance/verifiers/co-performance.sh b/.github/wai-conformance/verifiers/co-performance.sh new file mode 100644 index 000000000..111117a3e --- /dev/null +++ b/.github/wai-conformance/verifiers/co-performance.sh @@ -0,0 +1,27 @@ +# step: co-performance · wai-rs and co_performance_verify.py +# order: 510 +# features: co_performance +# paths: wai/extensions/co-performance.md +# +# wai.live.coperformance (extensions/co-performance.md): the reference module's unit tests +# (the policy and every statement round-trip and their rules refuse, the attributes' +# decimal integers, a schedule's turns in bars and microseconds, the symbolic bodies read +# whole) and tests/co_performance.rs (vectors.json regenerates byte for byte from its fixed +# seeds and no other file is there, wai-rs gives every room its stated outcome, and §10 +# lists exactly the reference's codes, each of which the corpus gives); then the corpus's +# second implementation, co_performance_verify.py, written from the extension's text and +# sharing no code with wai-rs, which verifies each room with live_rooms_verify.py and every +# signature with WAI's strict Ed25519 rule from claim-conformance, recomputes every intent, +# fidelity statement, turn, floor, commitment, selection and credit, reads the extension's +# reason table, requires every mutated copy of the corpus to fail, and drops each of its +# own checks in turn and requires the corpus to catch the weakened verifier; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/co-performance.md: tests/co_performance.rs requires this extension's §10 to list +# the codes the reference reports, co_performance_verify.py reads that table, and the registry's +# wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib co_performance::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test co_performance) +python3 -I live-conformance/co_performance_verify.py live-conformance/coperformance diff --git a/.github/wai-conformance/verifiers/co-watch.sh b/.github/wai-conformance/verifiers/co-watch.sh new file mode 100644 index 000000000..949b6db3e --- /dev/null +++ b/.github/wai-conformance/verifiers/co-watch.sh @@ -0,0 +1,30 @@ +# step: co-watch · wai-rs, co_watch_verify.py and the page's module +# order: 580 +# features: co_watch +# paths: wai/extensions/co-watch.md +# +# wai.live.cowatch (extensions/co-watch.md): the reference module's unit tests (the policy +# and its rules, statements read back and refused, the controlled position in the order of +# §5.4, each branch of the catch-up rule, percentiles by nearest rank, the control attribute) +# and tests/co_watch.rs (vectors.json regenerates byte for byte from its fixed seeds and no +# other file is there, wai-rs gives every room and catch-up case its stated outcome, and §12 +# lists exactly the reference's codes, each of which the corpus gives); then the corpus's +# second implementation, co_watch_verify.py, written from the extension's text and sharing +# no code with wai-rs, which verifies each room with live_rooms_verify.py and every +# signature with WAI's strict Ed25519 rule, recomputes the controlled position, every +# sample, percentile and verdict, requires every mutated copy of the corpus to fail and +# drops each of its own checks in turn and requires the corpus to catch the weakened +# verifier; standard library only, run isolated (-I). Last, the browser client's co-watch +# module (joule-moq-web/web/room/cowatch.js), which gives every catch-up case its +# correction, recomputes each accepted room's controlled position, and writes the corpus's +# controls and presentation statements byte for byte. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/co-watch.md: tests/co_watch.rs requires this extension's §12 to list the codes +# the reference reports, co_watch_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib co_watch::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test co_watch) +python3 -I live-conformance/co_watch_verify.py live-conformance/cowatch +node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/room/cowatch.test.mjs diff --git a/.github/wai-conformance/verifiers/conformance-results.sh b/.github/wai-conformance/verifiers/conformance-results.sh new file mode 100644 index 000000000..a1fbaca30 --- /dev/null +++ b/.github/wai-conformance/verifiers/conformance-results.sh @@ -0,0 +1,29 @@ +# step: conformance results · wai-rs and conform_result_verify.py +# order: 170 +# features: conform_result +# paths: wai/extensions/conformance-results.md +# +# conform-result/1 (extensions/conformance-results.md): the reference module's unit tests +# (a signed record accepted and every member covered, the strict rule's small-order forgery +# refused, the date form, a vector for every reason) and tests/conform_result.rs (the +# committed corpus regenerates byte for byte, wai-rs gives every vector its stated outcome, +# every reason of §5 refuses a vector, §5 lists exactly the reference's codes, and no leaf +# of a record survives a change after signing); then wai_conform_result_vectors verify, and +# the corpus's second implementation, conform_result_verify.py, written from the +# extension's text and sharing no code with wai-rs (WAI's strict Ed25519 rule from +# claim-conformance/_ed25519.py), which reads the extension's reason table; and with each +# check of §4 left out in turn (--skip), the corpus must fail it. Standard library only, +# run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/conformance-results.md: tests/conform_result.rs requires this extension's §5 to +# list the codes the reference reports, conform_result_verify.py reads that table, and the +# registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib conform_result::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test conform_result) +"$RUNNER_TEMP/wai-bins/union/wai_conform_result_vectors" verify conform-result-conformance +python3 -I conform-result-conformance/conform_result_verify.py conform-result-conformance +for c in duplicates format members signature summary verdict; do + python3 -I conform-result-conformance/conform_result_verify.py --skip "$c" conform-result-conformance | tail -1 +done diff --git a/.github/wai-conformance/verifiers/contribution.sh b/.github/wai-conformance/verifiers/contribution.sh new file mode 100644 index 000000000..c21c0087e --- /dev/null +++ b/.github/wai-conformance/verifiers/contribution.sh @@ -0,0 +1,28 @@ +# step: contribution records · wai-rs and contribution_verify.py +# order: 370 +# features: contribution +# paths: wai/extensions/contribution.md +# +# wai.media.contribution (extensions/contribution.md): the reference module's unit tests +# (every proof of the steps' tree walks to its root, the copying operations follow their +# sample mappings, a map has one encoding) and tests/contribution.rs (the committed corpus +# regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives +# every case its stated outcome and statements, §11 lists exactly the reference's codes, +# each of which the corpus gives, and the corpus generates with the model file of +# wai.generate.spectral's conformance vector); then the corpus's second implementation, +# contribution_verify.py, written from the extension's text and sharing no code with +# wai-rs, which evaluates every pinned operation, runs the generator in its own arithmetic +# after checking it against the generator's conformance vector, computes every map, share +# and C2PA statement itself, reads origin claims with synthesis_verify.py and every record +# with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, requires every mutated +# copy of the corpus to fail, and drops each of its own checks in turn and requires the +# corpus to catch the weakened verifier; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/contribution.md: tests/contribution.rs requires this extension's §11 to list the +# codes the reference reports, contribution_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib contribution::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test contribution) +python3 -I live-conformance/contribution_verify.py live-conformance/contribution diff --git a/.github/wai-conformance/verifiers/cue-injection-ertmp.sh b/.github/wai-conformance/verifiers/cue-injection-ertmp.sh new file mode 100644 index 000000000..7e29b9e2b --- /dev/null +++ b/.github/wai-conformance/verifiers/cue-injection-ertmp.sh @@ -0,0 +1,18 @@ +# step: cue injection and Enhanced RTMP · injection_verify.py and ertmp_verify.py +# order: 690 +# +# wai.live.injection (extensions/cue-injection.md) and Enhanced RTMP ingest +# (extensions/live-ingest.md §12): both corpora are generated through joule-moq-ingest, +# whose tests/conformance_corpus.rs regenerates them byte for byte +# (joule-code-conformance.yml). Here their second implementations, written from the +# texts and sharing no code with the reference: injection_verify.py translates every +# SCTE 104 message to its SCTE 35 sections and cues (the standard's sample among them, +# byte for byte), requires every refusal's result code and reason, runs the injector's +# rules over every scenario, verifies every chain with WAI's strict Ed25519 and derives +# its breaks, and refuses every negative chain at its event; ertmp_verify.py reads every +# RTMP message of every session and requires every object of every room track and what +# was not carried. Each then requires every mutated copy of its corpus to fail +# (--self-test). Standard library only, run isolated (-I). +set -euo pipefail +python3 -I live-conformance/injection_verify.py --self-test live-conformance/injection +python3 -I live-conformance/ertmp_verify.py --self-test live-conformance/ertmp diff --git a/.github/wai-conformance/verifiers/delivery-sim.sh b/.github/wai-conformance/verifiers/delivery-sim.sh new file mode 100644 index 000000000..8cc7ec662 --- /dev/null +++ b/.github/wai-conformance/verifiers/delivery-sim.sh @@ -0,0 +1,7 @@ +# step: sim_verify.py +# order: 80 +# +# The delivery simulator's second implementation: Appendix B from the text, every case +# of staged-measure-conformance/sim/ reproduced and its claim checked, with a +# counter-case required for each part of the stall bound a schedule can lose. +python3 staged-measure-conformance/sim_verify.py staged-measure-conformance diff --git a/.github/wai-conformance/verifiers/deriving-edge.sh b/.github/wai-conformance/verifiers/deriving-edge.sh new file mode 100644 index 000000000..e0278f76c --- /dev/null +++ b/.github/wai-conformance/verifiers/deriving-edge.sh @@ -0,0 +1,30 @@ +# step: deriving edge · wai-rs and edge_verify.py +# order: 680 +# features: derive_edge +# paths: wai/extensions/deriving-edge.md +# +# wai.edge.derivation (extensions/deriving-edge.md): the reference module's unit tests +# (the operations pixel by pixel, which operations the registry's composition rule makes +# exact, the check each derivation takes, a record's reading rule by rule, its round trip +# and signature, the pinned output manifest, provenance items and AI-origin labels, the +# C2PA stores embedded in a still), the still writer and reader the module +# shares with wai-pack (png_still), and tests/derive_edge.rs (vectors.json regenerates +# byte for byte from its fixed seeds and no other file is there, wai-rs gives every vector +# its stated outcome, §9 lists exactly the reference's codes and §4 its derivations, and +# the reference edge wai_edge, run twice over a clip, makes a chain of two hops that +# verifies and is refused once a byte changes); then the corpus's second implementation, +# edge_verify.py, written from the extension's text and sharing no code with wai-rs, +# which reads receipts and writes stills itself, re-executes every replicate hop, +# requires every mutated copy of the corpus to fail, and drops each of its own checks in +# turn and requires the corpus to catch the weakened verifier; standard library only, +# run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/deriving-edge.md: tests/derive_edge.rs requires this extension's §9 to list the +# codes the reference reports and its §4 the derivations, edge_verify.py reads both tables, and +# the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib derive_edge::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib png_still::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test derive_edge) +python3 -I edge-conformance/edge_verify.py edge-conformance diff --git a/.github/wai-conformance/verifiers/determinism-cost.sh b/.github/wai-conformance/verifiers/determinism-cost.sh new file mode 100644 index 000000000..81df4f2ad --- /dev/null +++ b/.github/wai-conformance/verifiers/determinism-cost.sh @@ -0,0 +1,15 @@ +# step: determinism cost · the record, the text quoting it, and wai_det_cost +# order: 20 +# features: neural_int +# +# The determinism-cost record (research/determinism-cost.md): its summary recomputed +# from its readings by a second implementation that shares no code with the tool that +# wrote it (tools/wai_determinism_cost_verify.py, standard library only, run isolated), +# and every figure the research page and determinism-tiers §7 quote from it checked +# against it (each is followed by a `det` marker naming its place in the record). Then +# wai_det_cost, the decoder the record ran, held to the committed int_hyper vector and +# every WIH1 refusal fixture of int-payload-conformance/. +set -euo pipefail +python3 -I tools/wai_determinism_cost_verify.py research/determinism-cost/record.json \ + research/determinism-cost.md extensions/determinism-tiers.md +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test det_cost_bin) diff --git a/.github/wai-conformance/verifiers/direction.sh b/.github/wai-conformance/verifiers/direction.sh new file mode 100644 index 000000000..f4f78b6fb --- /dev/null +++ b/.github/wai-conformance/verifiers/direction.sh @@ -0,0 +1,29 @@ +# step: direction records · wai-rs and direction_verify.py +# order: 500 +# features: direction +# paths: wai/extensions/direction.md +# +# wai.media.direction (extensions/direction.md): the reference module's unit tests (the +# interpolations reach both keys and stay between them, the framing measure at its bounds, +# a signed integer's one spelling, the timeline's starts and overlaps, a frame set's one +# encoding) and tests/direction.rs (the committed corpus regenerates byte for byte from its +# fixed seeds and no other file is there, wai-rs gives every case its stated outcome and +# statements, §13 lists exactly the reference's codes, each of which the corpus gives, and +# the corpus generates with the model file of wai.generate.spectral's conformance vector); +# then the corpus's second implementation, direction_verify.py, written from the +# extension's text and sharing no code with wai-rs, which runs the generator in its own +# arithmetic after checking it against the generator's conformance vector, renders every +# frame, interpolates every track, measures every box and writes every statement and C2PA +# statement itself, reads records with synthesis_verify.py's strict JSON reader and every +# signature with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, requires every +# mutated copy of the corpus to fail, and drops each of its own checks in turn and requires +# the corpus to catch the weakened verifier; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/direction.md: tests/direction.rs requires this extension's §13 to list the codes +# the reference reports, direction_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib direction::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test direction) +python3 -I live-conformance/direction_verify.py live-conformance/direction diff --git a/.github/wai-conformance/verifiers/e2ee.sh b/.github/wai-conformance/verifiers/e2ee.sh new file mode 100644 index 000000000..9020d0ddb --- /dev/null +++ b/.github/wai-conformance/verifiers/e2ee.sh @@ -0,0 +1,19 @@ +# step: end-to-end encrypted rooms · e2ee_verify.py +# order: 650 +# paths: wai/extensions/confidentiality.md +# +# wai.live.e2ee (extensions/confidentiality.md): the corpus's second implementation, +# e2ee_verify.py, written from the extension's text and the documents it reads and +# sharing no code with the reference (joule-moq's secure_objects and joule-moq-mls, +# whose own checks run in joule-code-conformance.yml). Standard library only, run +# isolated (-I): AES, AES-GCM and RFC 9605's counter mode written from their +# specifications for test vectors only, HKDF and HMAC from the standard library, +# BLAKE3 and strict Ed25519 from claim-conformance. It checks every vector RFC 9605 +# and draft-ietf-moq-secure-objects-01 publish, every record, key schedule, sealed +# object and room of vectors.json (taking each epoch's application secret as given, +# §14), and requires every mutated copy of the corpus to fail. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/confidentiality.md: e2ee_verify.py reads this extension's §13 table, and the +# registry's wire identifiers test reads its sections. +python3 -I live-conformance/e2ee_verify.py live-conformance/e2ee diff --git a/.github/wai-conformance/verifiers/energy-steering.sh b/.github/wai-conformance/verifiers/energy-steering.sh new file mode 100644 index 000000000..8a1ccbed8 --- /dev/null +++ b/.github/wai-conformance/verifiers/energy-steering.sh @@ -0,0 +1,27 @@ +# step: energy steering · wai-rs and energy_steering_verify.py +# order: 190 +# features: energy_steering +# paths: wai/extensions/energy-steering.md +# +# wai.live.steering (extensions/energy-steering.md): the reference module's unit tests +# (policies and decisions round-trip and their rules refuse, the window, the request +# reader over HLS, DASH and CMCD, the RFC 9651 parser, exact products at their limits, +# the margin at its boundary, figures compared only within a class, and a pathway with +# no figure never placed first by its rank while one with a figure is within the +# limits) and tests/energy_steering.rs (the committed corpus regenerates byte for byte +# from its fixed seeds, wai-rs gives every request and vector its stated outcome, every +# reason of §10 refuses a vector, and §10 lists exactly the reference's codes); then the +# corpus's second implementation, energy_steering_verify.py, written from the +# extension's text and sharing no code with wai-rs, which reads the room with +# live_rooms_verify.py and the per-hop records with live_metering_verify.py, parses RFC +# 9651 itself, recomputes every report, figure, decision and manifest, and requires +# every mutated copy of the corpus to fail; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/energy-steering.md: tests/energy_steering.rs requires this extension's §10 to +# list the codes the reference reports, energy_steering_verify.py reads that table, and the +# registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib energy_steering::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test energy_steering) +python3 -I live-conformance/energy_steering_verify.py live-conformance/steering diff --git a/.github/wai-conformance/verifiers/exact-splats.sh b/.github/wai-conformance/verifiers/exact-splats.sh new file mode 100644 index 000000000..a6d4a813f --- /dev/null +++ b/.github/wai-conformance/verifiers/exact-splats.sh @@ -0,0 +1,30 @@ +# step: exact splats · wai-rs and splat_exact_verify.py +# order: 620 +# features: splat_exact +# paths: wai/extensions/splat-exact.md wai/extensions/gltf/** +# +# wai.splat.exact (extensions/splat-exact.md): the reference module's unit tests +# (exp_neg_q16 against the exponential, binary32 ranges rounded half up exactly, the +# signed integers' grammar, the identity set carrying every byte, and tiles that leave a +# frame unchanged) and tests/splat_exact.rs (live-conformance/splat/ regenerates byte for +# byte and no other file is there, every case gets its stated outcome, §10 lists exactly the +# reference's codes, the tiles leave every view of every scene unchanged, and the draft +# schema names §3.2's members); then the corpus's second implementation, +# splat_exact_verify.py, written from the extension's text and sharing no code with wai-rs: +# it reads each glTF asset, decodes each payload with int_payload_verify.py's parser and +# rANS decoder and tools/wai_ops_check.py's whm1/1 interpreter, renders every view with its +# own integer renderer, verifies every record with WAI's strict Ed25519 rule and BLAKE3 +# from claim-conformance, reads the extension's reason table, requires every mutated copy +# of the corpus to fail, drops each of its own checks in turn and changes each rule of its +# renderer in turn, and requires the corpus to catch every one; standard library only, run +# isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/splat-exact.md and wai/extensions/gltf/**: tests/splat_exact.rs requires this +# extension's §10 to list the codes the reference reports and the draft schema under +# extensions/gltf/ to name §3.2's members, splat_exact_verify.py reads §10, and the registry's +# wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib splat_exact::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test splat_exact) +python3 -I live-conformance/splat_exact_verify.py live-conformance/splat diff --git a/.github/wai-conformance/verifiers/exact-tonemap.sh b/.github/wai-conformance/verifiers/exact-tonemap.sh new file mode 100644 index 000000000..6b8a36417 --- /dev/null +++ b/.github/wai-conformance/verifiers/exact-tonemap.sh @@ -0,0 +1,30 @@ +# step: exact tone mapping · wai-rs and tonemap_verify.py +# order: 610 +# features: exact_tonemap +# paths: wai/extensions/exact-tonemap.md +# +# wai.media.tonemap, wai.meta.gainmap and wai.meta.st2094_50 (extensions/exact-tonemap.md): +# the reference module's unit tests (the fixed-point log2 and exp2 at known points, the +# table of K against its rule, the sRGB table, tan through its series, PCHIP slopes of a +# line, rounding of halves for both signs) and the PNG reader's, then tests/exact_tonemap.rs +# (the committed corpus regenerates byte for byte, no other file is there, every case and +# envelope gets its stated outcome, §9 lists exactly the reference's codes, §6 states the +# corpus's measurement against binary64, and frames.rgb8 is the int_video clip's decode +# whose digests its sidecar publishes); then the corpus's second implementation, +# tonemap_verify.py, written from the extension's text and sharing no code with wai-rs, +# which computes both derivations in integer arithmetic of its own, reads PNG, envelopes +# (with staged_verify.py) and both companions itself, recomputes §6's measurement, verifies +# every record with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, requires +# every mutated copy of the corpus to fail, and drops each of its own checks in turn and +# requires the corpus to catch the weakened verifier; standard library only, run isolated +# (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/exact-tonemap.md: tests/exact_tonemap.rs requires this extension's §9 to list the +# codes the reference reports and its §6 to state the corpus's measurement, tonemap_verify.py +# reads §3.3's table, §5.4's constant and §9, and the registry's wire identifiers test reads its +# sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- exact_tonemap:: png_still::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test exact_tonemap) +python3 -I live-conformance/tonemap_verify.py live-conformance/tonemap diff --git a/.github/wai-conformance/verifiers/exact-upscale.sh b/.github/wai-conformance/verifiers/exact-upscale.sh new file mode 100644 index 000000000..bc2d9ceed --- /dev/null +++ b/.github/wai-conformance/verifiers/exact-upscale.sh @@ -0,0 +1,30 @@ +# step: exact upscale · wai-rs and upscale_verify.py +# order: 310 +# features: exact_upscale neural_int +# paths: wai/extensions/exact-upscale.md +# +# wai.media.upscale (extensions/exact-upscale.md): the integer super-resolution +# derivations' unit tests (the wai-ops/1 graph against the arithmetic of §3.2 written out +# loop by loop, on random priors and pictures; a prior with no detail is the pinned +# upscale; sr4 is sr2 twice; a prior that breaks §3.1 refused in its order; a picture past +# the caps refused), the module's unit tests (the pixel counts of §6.3, a residual within +# half its step), and tests/exact_upscale.rs (the committed corpus regenerates byte for +# byte from its fixed seeds and its two pinned trained priors, and no other file is there; +# wai-rs gives every case its stated outcome and statements; §8 lists exactly the +# reference's codes, each of which the corpus gives; sha256.json names every accepted +# output); then the corpus's second implementation, upscale_verify.py, written from the +# extension's text and sharing no code with wai-rs, which runs the derivations with +# _int_sr.py, verifies the claims with synthesis_verify.py and the residuals with +# staged_verify.py, and requires every mutated copy of the corpus to fail; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/exact-upscale.md: tests/exact_upscale.rs requires this extension's §8 to list the +# codes the reference reports, upscale_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib int_sr::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib exact_upscale::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test exact_upscale) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test cli_derive) +python3 -I live-conformance/upscale_verify.py live-conformance/upscale diff --git a/.github/wai-conformance/verifiers/exploration.sh b/.github/wai-conformance/verifiers/exploration.sh new file mode 100644 index 000000000..570078031 --- /dev/null +++ b/.github/wai-conformance/verifiers/exploration.sh @@ -0,0 +1,29 @@ +# step: exploration · wai-rs and exploration_verify.py +# order: 560 +# features: exploration +# paths: wai/extensions/exploration.md +# +# wai.media.exploration (extensions/exploration.md): the reference module's unit tests (the +# descriptors of flat, single-row and checkerboard pictures, grid bins at both ends, the +# sample distance's zero, the taste score, derived prompts, and a schedule that replays and +# keeps the first comer without an objective) and tests/exploration.rs (vectors.json +# regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives +# every generation, descriptor vector and case its stated outcome, §13 lists exactly the +# reference's codes, each of which the corpus gives, and §3.2 names the reference's +# descriptors); then the corpus's second implementation, exploration_verify.py, written from +# the extension's text and sharing no code with wai-rs, which makes every picture with the +# generator as the extension's Appendix A restates it, computes the descriptors, replays +# every archive's schedule, reads usage statements with ai_usage_verify.py, verifies every +# record with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, reads the +# extension's reason table, requires every mutated copy of the corpus to fail, and drops each +# of its own checks in turn and requires the corpus to catch the weakened verifier; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/exploration.md: tests/exploration.rs requires this extension's §13 to list the +# codes the reference reports and its §3.2 table to name the reference's descriptors, +# exploration_verify.py reads §13, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib exploration::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test exploration) +python3 -I live-conformance/exploration_verify.py live-conformance/exploration diff --git a/.github/wai-conformance/verifiers/fleet.sh b/.github/wai-conformance/verifiers/fleet.sh new file mode 100644 index 000000000..6fa355fb6 --- /dev/null +++ b/.github/wai-conformance/verifiers/fleet.sh @@ -0,0 +1,28 @@ +# step: fleet · wai-rs and fleet_verify.py +# order: 440 +# features: fleet +# paths: wai/extensions/fleet.md +# +# wai.delivery.fleet (extensions/fleet.md): the reference module's unit tests (the enrolment's +# rules, a summary line split only in the relay's form, a row summed across classes told from +# a wrong one, the four coverages) and tests/fleet.rs (vectors.json regenerates byte for byte +# from its seeds and the metering corpus and no other file is there; wai-rs gives every +# vector its stated outcome; every reason of §10 and every reused code refuses a vector; the +# accepted vectors state measured, unmetered and partial energy as §5.4 does; the week's +# records held as a service holds them sign the corpus's report byte for byte; §10 lists +# exactly the reference's codes); then the corpus's second implementation, fleet_verify.py, +# written from the extension's text and sharing no code with wai-rs, which reads per-hop +# records with live_metering_verify.py, checks every signature with WAI's strict Ed25519 +# rule, recomputes every line, row, unmetered count and citation, requires every mutated +# copy of the corpus to fail, and drops each of its own checks in turn and requires the +# corpus to catch the weakened verifier; standard library only, run isolated (-I). The +# service and its smoke checks over HTTP run in wai-fleet.yml. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/fleet.md: tests/fleet.rs requires this extension's §10 to list the codes the +# reference reports, fleet_verify.py reads that table, and the registry's wire identifiers test +# reads its sections. Its corpus is generated from the metering corpus, under the paths above. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib fleet::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test fleet) +python3 -I live-conformance/fleet_verify.py live-conformance/fleet diff --git a/.github/wai-conformance/verifiers/forensic-watermark.sh b/.github/wai-conformance/verifiers/forensic-watermark.sh new file mode 100644 index 000000000..47da6ccfd --- /dev/null +++ b/.github/wai-conformance/verifiers/forensic-watermark.sh @@ -0,0 +1,26 @@ +# step: forensic watermark sessions · wai-rs and live_watermark_verify.py +# order: 670 +# features: live_watermark +# paths: wai/extensions/forensic-watermark.md +# +# wai.live.watermark (extensions/forensic-watermark.md): the reference module's unit tests +# (the CBOR reader's deterministic-encoding rules, tokens of both COSE structures and +# every mode read and their rules refuse, a record's rules, big-endian bit addressing, +# the exact match and the binding) and tests/live_watermark.rs (vectors.json regenerates +# byte for byte from its fixed seeds and no other file is there, wai-rs gives every +# token, session list, match and disclosure its stated outcome, and §11 lists exactly the +# reference's codes, each of which the corpus gives); then the corpus's second +# implementation, live_watermark_verify.py, written from the extension's text and sharing +# no code with wai-rs, which reads rooms with live_rooms_verify.py and tokens with a CBOR +# reader of its own, requires every mutated copy of the corpus to fail, and drops each of +# its own checks in turn and requires the corpus to catch the weakened verifier; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/forensic-watermark.md: tests/live_watermark.rs requires this extension's §11 to +# list the codes the reference reports, live_watermark_verify.py reads that table, and the +# registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_watermark::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_watermark) +python3 -I live-conformance/live_watermark_verify.py live-conformance/watermark diff --git a/.github/wai-conformance/verifiers/header-probes.sh b/.github/wai-conformance/verifiers/header-probes.sh new file mode 100644 index 000000000..9b50e55e9 --- /dev/null +++ b/.github/wai-conformance/verifiers/header-probes.sh @@ -0,0 +1,29 @@ +# step: header probes · wai-rs, the fetched LCEVC streams and format_probe_verify.py +# order: 730 +# features: format_probe +# paths: wai/extensions/format-probes.md +# +# Header probes (extensions/format-probes.md, projection.md): the reference module's unit +# tests (bits, Exp-Golomb and emulation prevention, the codes) and tests/format_probe.rs (the +# committed corpus regenerates byte for byte from its external vectors and holds no other +# generated file, every vector, binding and fact gets its stated outcome, and the +# extensions' tables list exactly the reference's probes and codes); then the five LCEVC +# streams fetch.sh fetches and checks by SHA-256, each of which wai_format_probe and the +# second implementation require (a missing one is a failure); then +# format_probe_verify.py, written from the two texts and sharing no code with wai-rs, +# which probes every vector itself and requires every mutated copy of the corpus to fail. +# Standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/format-probes.md: tests/format_probe.rs requires format-probes §2.4 and §12 to +# list the reference's probes and codes, and projection.md §4 and §5 to state its codes; +# format_probe_verify.py reads format-probes' tables; and the registry's wire identifiers test +# reads projection.md, loc-carriage.md and delivery-telemetry.md (whose §6.6 cmsd_read_verify.py +# reads). +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib format_probe::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test format_probe) +fetched=$(mktemp -d) +bash format-probe-conformance/fetch.sh "$fetched" +(cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_format_probe" verify ../format-probe-conformance --fetched "$fetched") +python3 -I format-probe-conformance/format_probe_verify.py format-probe-conformance --fetched "$fetched" diff --git a/.github/wai-conformance/verifiers/holdings.sh b/.github/wai-conformance/verifiers/holdings.sh new file mode 100644 index 000000000..16804314f --- /dev/null +++ b/.github/wai-conformance/verifiers/holdings.sh @@ -0,0 +1,22 @@ +# step: holdings · wai-rs and holdings_verify.py +# order: 550 +# features: holdings +# paths: wai/extensions/holdings.md +# +# wai.delivery.holdings (extensions/holdings.md): the reference module's unit tests (an entry's +# one form, a record read back and every rule refusing, queries and a plan, and the corpus +# reaching every stated outcome and failing a changed one) and tests/holdings.rs (vectors.json +# regenerates byte for byte from its fixed seeds, wai-rs gives every case its stated outcome, +# and §7 lists exactly the reference's codes); then the corpus's second implementation, +# holdings_verify.py, written from the extension's text and sharing no code with wai-rs, which +# verifies group receipts with jwp_verify.py and signatures with the shared _ed25519.py, writes +# the record, its verdict, queries and plans itself, reads §7, requires every mutated copy of +# the corpus to fail and drops each of its own checks in turn; run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/holdings.md: tests/holdings.rs requires §7 to list the reference's codes, and +# holdings_verify.py reads it. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- holdings::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test holdings) +python3 -I live-conformance/holdings_verify.py live-conformance/holdings diff --git a/.github/wai-conformance/verifiers/http-differential.sh b/.github/wai-conformance/verifiers/http-differential.sh new file mode 100644 index 000000000..887450b08 --- /dev/null +++ b/.github/wai-conformance/verifiers/http-differential.sh @@ -0,0 +1,19 @@ +# step: The HTTP differential outside the corpus (wai-rs against http_verify.py) +# order: 60 +# features: http,video_manifest,key_discovery +# +# Outside the corpus: wai_http_vectors fuzz writes seeded cases (Content-Range values, +# codecs values, #WAI-LAYER lines, manifests and their member types, encryption, +# refines, layers declared against their bases, layers over WAI2 components, objects +# past the first request through a tampering origin; playlists, adjacent pairs, +# published masters, renderings, adaptation states, the integer log and changed +# receipt files; receipt files, indexes, grants files and key logs of §8.1–§8.5, +# whose cases need the `cryptography` package installed above) with wai-rs's +# outcomes, and +# http_verify.py must give every one, a refusal by the same code. It prints each +# family's count of cases and of refusals. Two fixed seeds, so a run is reproducible. +set -euo pipefail +for seed in 11 12; do + "$RUNNER_TEMP/wai-bins/union/wai_http_vectors" fuzz http-conformance --seed "$seed" --count 400 --out "$RUNNER_TEMP/http-fuzz-$seed.json" + python3 http-conformance/http_verify.py cases "$RUNNER_TEMP/http-fuzz-$seed.json" http-conformance +done diff --git a/.github/wai-conformance/verifiers/http.sh b/.github/wai-conformance/verifiers/http.sh new file mode 100644 index 000000000..5e7937695 --- /dev/null +++ b/.github/wai-conformance/verifiers/http.sh @@ -0,0 +1,15 @@ +# step: http_verify.py +# order: 50 +# +# The HTTP delivery corpus's second implementation (http-conformance/README): the head, +# the head proof (every committed proof rebuilt from the text), response checks, plans, +# selection, the byte-range procedure and its fallback, the codecs union and floor, the +# playlists (read, rendered, held adjacent, the classical master unchanged), playlist +# receipts (WAI's strict Ed25519, _ed25519.py) and the live edge, and the adaptation policies, +# checked first against published BLAKE3 vectors, with the standard library alone; +# and the receipts, key logs, grants files and parameter sets of §8.1–§8.5, through +# http_receipts.py and the corpora's own verifiers of jwp-receipts, which need the +# `cryptography` package installed above (without it those cases fail, never skip). +# Its mutations and the differential against wai-rs run in wai-http.yml's falsify +# job. +python3 http-conformance/http_verify.py http-conformance diff --git a/.github/wai-conformance/verifiers/hybrid-signatures.sh b/.github/wai-conformance/verifiers/hybrid-signatures.sh new file mode 100644 index 000000000..3279f8b93 --- /dev/null +++ b/.github/wai-conformance/verifiers/hybrid-signatures.sh @@ -0,0 +1,31 @@ +# step: hybrid signatures · wai-rs and pq_verify.py +# order: 710 +# build: pq +# features: pq_signatures synthesis,pq_signatures +# paths: wai/extensions/pq-signatures.md +# +# Hybrid signatures (extensions/pq-signatures.md): the reference module's unit tests (its +# ML-DSA-65 against FIPS 204's ACVP vectors — keyGen, deterministic sigGen and sigVer — +# vendored in pq-conformance/acvp and pinned by SHA-256, and the composite against the +# draft's own vector, before anything is built on them), the synthesis reader's hybrid +# records, and tests/pq_signatures.rs (vectors.json regenerates byte for byte, every case +# reaches its verdict, §8 lists exactly the reference's codes, each given); then the +# corpus's second implementation, pq_verify.py, written from the extension's text and +# sharing no code with wai-rs, which verifies ML-DSA-65 from FIPS 204's text in pure +# Python (_mldsa.py), checks every ACVP verification vector and signature and the draft's +# vector, every case, the sizes §9 states and §8's table, requires every mutated case to be +# refused and the corpus to catch it with any one of its checks dropped; run isolated (-I). +# Last, the two give every case the same verdict. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/pq-signatures.md: tests/pq_signatures.rs requires this extension's §8 to list the +# codes the reference reports, pq_verify.py reads that table, and the registry's wire +# identifiers test reads its §2. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_PQ_FEATURES" --lib pq_sig::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_PQ_FEATURES" --lib synthesis::pq_tests) +(cd wai-rs && cargo test --no-default-features --features "$WAI_PQ_FEATURES" --test pq_signatures) +python3 -I pq-conformance/pq_verify.py pq-conformance +python3 -I pq-conformance/pq_verify.py verdicts pq-conformance/vectors.json > "$RUNNER_TEMP/pq-py.tsv" +(cd wai-rs && "$RUNNER_TEMP/wai-bins/pq/wai_pq_vectors" verdicts ../pq-conformance/vectors.json) > "$RUNNER_TEMP/pq-rs.tsv" +diff "$RUNNER_TEMP/pq-py.tsv" "$RUNNER_TEMP/pq-rs.tsv" diff --git a/.github/wai-conformance/verifiers/influence.sh b/.github/wai-conformance/verifiers/influence.sh new file mode 100644 index 000000000..843e672e8 --- /dev/null +++ b/.github/wai-conformance/verifiers/influence.sh @@ -0,0 +1,28 @@ +# step: influence attribution · wai-rs and influence_verify.py +# order: 490 +# features: influence +# paths: wai/extensions/influence.md +# +# wai.media.influence (extensions/influence.md): the reference module's unit tests (the +# blend keeps its base and its references by weight, the makers' split is whole, an +# estimate mixed into a share is told from another difference) and tests/influence.rs (the +# committed corpus regenerates byte for byte from its fixed seeds and no other file is +# there, wai-rs gives every case, statement and report its stated outcome, every reason of +# §9 and every code it reuses refuses a vector, and §9 lists exactly the reference's codes); +# then the corpus's second implementation, influence_verify.py, written from the +# extension's text and sharing no code with wai-rs, which verifies each session with +# contribution_verify.py, regenerates every conditioned step with that implementation's +# arithmetic of the generator, reads usage statements with ai_usage_verify.py and +# statements with settlement_verify.py, computes every step map, share, attribution line +# and report itself, requires every mutated copy of the corpus to fail, and drops each of +# its own checks in turn and requires the corpus to catch the weakened verifier; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/influence.md: tests/influence.rs requires this extension's §9 to list the codes +# the reference adds, influence_verify.py reads that table, and the registry's wire identifiers +# test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib influence::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test influence) +python3 -I live-conformance/influence_verify.py live-conformance/influence diff --git a/.github/wai-conformance/verifiers/int-payload.sh b/.github/wai-conformance/verifiers/int-payload.sh new file mode 100644 index 000000000..b80b73a1d --- /dev/null +++ b/.github/wai-conformance/verifiers/int-payload.sh @@ -0,0 +1,9 @@ +# step: int_payload_verify.py +# order: 10 +# +# The integer payload corpus's second implementation (int-payload-conformance/README): +# it parses every payload with the extension's rules and check order, converts every +# JSON source independently and requires the committed bytes, decodes the WIV1 +# keyframe-0 payloads in full to their sidecars' SHA-256, and checks every refusal +# fixture. Standard library only, so it needs nothing installed. +python3 int-payload-conformance/int_payload_verify.py int-payload-conformance diff --git a/.github/wai-conformance/verifiers/key-release.sh b/.github/wai-conformance/verifiers/key-release.sh new file mode 100644 index 000000000..c0684a4ac --- /dev/null +++ b/.github/wai-conformance/verifiers/key-release.sh @@ -0,0 +1,28 @@ +# step: key release · wai-rs and key_release_verify.py +# order: 240 +# features: key_release +# paths: wai/extensions/key-release.md +# +# The proxy form of wai.video.keys (extensions/key-release.md): the reference module's unit +# tests (the commitment list, the entitlement and the ticket round-trip and their rules +# refuse, each check of the decision in its order, offline licences and renewals, each own +# rule of the receipt, its strict signature, the CPIX reader and the derivation of a list +# from a CPIX document), form 1's unit tests (its legacy pin unchanged), and +# tests/key_release.rs (every file of the committed corpus regenerates byte for byte from +# its fixed seeds and no other file is there, every log gets its stated outcome, and the +# extension's tables are the reference's codes); then the corpus's second implementation, +# key_release_verify.py, written from the extension's text and sharing no code with wai-rs, +# which reads the extension's tables, derives every list from its CPIX document with the +# standard library's XML parser, recomputes every decision of every log, verifies every +# signature with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, and requires +# every mutated copy of the corpus to fail; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/key-release.md: tests/key_release.rs requires this extension's §3.1, §3.2, §7.2 +# and §11 tables to be the reference's codes, key_release_verify.py reads those tables and §3.3, +# and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib key_release::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib wai_video_keys::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test key_release) +python3 -I live-conformance/key_release_verify.py live-conformance/keys diff --git a/.github/wai-conformance/verifiers/live-ads.sh b/.github/wai-conformance/verifiers/live-ads.sh new file mode 100644 index 000000000..36147566e --- /dev/null +++ b/.github/wai-conformance/verifiers/live-ads.sh @@ -0,0 +1,29 @@ +# step: live ads · wai-rs and live_ads_verify.py +# order: 230 +# features: live_ads +# paths: wai/extensions/live-ads.md +# +# wai.live.ads (extensions/live-ads.md): the reference module's unit tests (policies and +# annotations round-trip and their rules refuse, a break's identity, tags, events and +# asset lists written exactly), the break form of wai.video.ssai's receipt (its JSON +# carriage, each of its rules, its strict signature, the stitched form's legacy pin +# unchanged), and tests/live_ads.rs (every file of the committed corpus, vectors.json and +# each room's HLS tags, regenerates byte for byte from its fixed seeds and no other file +# is there, wai-rs gives every room its stated outcome, and §10 lists exactly the +# reference's codes, each of which the corpus gives); then the corpus's second +# implementation, live_ads_verify.py, written from the extension's text and sharing no +# code with wai-rs, which verifies each room with live_rooms_verify.py, writes every tag, +# event and asset list byte for byte, verifies every receipt with WAI's strict Ed25519 +# rule and BLAKE3 from claim-conformance, reads the extension's reason table, and +# requires every mutated copy of the corpus to fail; standard library only, run isolated +# (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-ads.md: tests/live_ads.rs requires this extension's §10 to list the codes +# the reference reports, live_ads_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_ads::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib wai_video_ssai::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_ads) +python3 -I live-conformance/live_ads_verify.py live-conformance/ads diff --git a/.github/wai-conformance/verifiers/live-compose.sh b/.github/wai-conformance/verifiers/live-compose.sh new file mode 100644 index 000000000..1ef18f2eb --- /dev/null +++ b/.github/wai-conformance/verifiers/live-compose.sh @@ -0,0 +1,25 @@ +# step: live compose · wai-rs and live_compose_verify.py +# order: 130 +# features: live_compose +# paths: wai/extensions/live-compose.md +# +# wai.live.compose (extensions/live-compose.md): the reference module's unit tests (the +# layout over every grid of 1 to 64 tiles stays on its canvas with no tile overlapping +# another, the fill modes, the attributes, the mixer) and tests/live_compose.rs (the +# committed corpus regenerates byte for byte from its fixed seeds, wai-rs gives every +# case its stated outcome, every reason of §10 refuses a case, and §10 lists exactly +# the reference's codes); then the corpus's second implementation, +# live_compose_verify.py, written from the extension's text and sharing no code with +# wai-rs, which verifies each room with live_rooms_verify.py, steps through its +# lifecycle replay to each instant itself, gives every layout, mix, descriptor, +# composition and receipt the same outcome rectangle for rectangle and sample for +# sample, and requires every mutated copy of the corpus to fail; standard library +# only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-compose.md: tests/live_compose.rs requires this extension's §10 to list the +# codes the reference reports, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_compose::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_compose) +python3 -I live-conformance/live_compose_verify.py live-conformance/compose diff --git a/.github/wai-conformance/verifiers/live-contribution.sh b/.github/wai-conformance/verifiers/live-contribution.sh new file mode 100644 index 000000000..176573612 --- /dev/null +++ b/.github/wai-conformance/verifiers/live-contribution.sh @@ -0,0 +1,19 @@ +# step: live contribution · live_contribution_verify.py +# order: 380 +# paths: wai/extensions/live-contribution.md +# +# wai.live.contribution (extensions/live-contribution.md): the link records' corpus +# under live-conformance/links/, which joule-moq-ingest's reference generates and its +# own tests check (joule-code-conformance.yml); here its second implementation, +# live_contribution_verify.py, written from the extension's text and sharing no code +# with the reference, reads every part and series itself, checks every signature with +# WAI's strict Ed25519 rule from claim-conformance, requires each vector's stated outcome +# and reason and every reason of §7's last list, requires every mutated copy to be +# refused, and drops each of its own checks in turn and requires the corpus to catch the +# weakened verifier; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-contribution.md: live_contribution_verify.py reads this extension's §7 list +# of the link record's reasons and requires the corpus to raise each one. +set -euo pipefail +python3 -I live-conformance/live_contribution_verify.py live-conformance/links diff --git a/.github/wai-conformance/verifiers/live-events.sh b/.github/wai-conformance/verifiers/live-events.sh new file mode 100644 index 000000000..5643a9efc --- /dev/null +++ b/.github/wai-conformance/verifiers/live-events.sh @@ -0,0 +1,21 @@ +# step: live events · wai-rs and live_events_verify.py +# order: 100 +# features: live_events +# paths: wai/extensions/live-events.md +# +# wai.live.events (extensions/live-events.md): the reference module's unit tests and +# tests/live_events.rs (the committed corpus regenerates byte for byte from its fixed +# seeds, wai-rs gives every vector, policy case and binding case its stated outcome, +# the corpus covers every kind and reason, and §9 lists exactly the reference's +# codes); then the corpus's second implementation, live_events_verify.py, written +# from the extension's text and sharing no code with wai-rs, which verifies the same +# cases with WAI's strict Ed25519 (claim-conformance/_ed25519.py) and BLAKE3 +# (claim-conformance/_blake3.py), standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-events.md: tests/live_events.rs requires this extension's §9 to list the +# codes the reference reports, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_events::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_events) +python3 -I live-conformance/live_events_verify.py live-conformance/events diff --git a/.github/wai-conformance/verifiers/live-messaging.sh b/.github/wai-conformance/verifiers/live-messaging.sh new file mode 100644 index 000000000..95eef65ed --- /dev/null +++ b/.github/wai-conformance/verifiers/live-messaging.sh @@ -0,0 +1,24 @@ +# step: live messaging · wai-rs and live_messaging_verify.py +# order: 660 +# features: live_messaging +# paths: wai/extensions/live-messaging.md +# +# wai.live.message and wai.live.call (extensions/live-messaging.md): tests/live_messaging.rs +# (vectors.json regenerates byte for byte from its fixed seeds and no other file is there, +# wai-rs gives every conversation, history request, call and record its stated outcome, and +# §12 lists exactly the reference's codes, each of which the corpus gives); then the corpus's +# second implementation, live_messaging_verify.py, written from the extension's text and +# sharing no code with wai-rs, which reads descriptors with live_rooms_verify.py and Key IDs, +# keys, AES-GCM and plaintexts with e2ee_verify.py, implements the receipts' binary form and +# every rule of the extension itself, verifies every signature with WAI's strict Ed25519 +# rule, requires every mutated copy of the corpus to fail, and drops each of its own checks +# in turn and requires the corpus to catch the weakened verifier; standard library only, run +# isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-messaging.md: tests/live_messaging.rs requires this extension's §12 to list +# the codes the reference reports, live_messaging_verify.py reads that table, and the registry's +# wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_messaging) +python3 -I live-conformance/live_messaging_verify.py live-conformance/messaging diff --git a/.github/wai-conformance/verifiers/live-metering.sh b/.github/wai-conformance/verifiers/live-metering.sh new file mode 100644 index 000000000..3426730f9 --- /dev/null +++ b/.github/wai-conformance/verifiers/live-metering.sh @@ -0,0 +1,27 @@ +# step: live metering · wai-rs and live_metering_verify.py +# order: 180 +# features: live_metering +# paths: wai/extensions/live-metering.md +# +# wai.live.metering (extensions/live-metering.md): the reference module's unit tests +# (records and ledgers round-trip, the energy rules, the size bound at its limit, a +# total of two classes told from a wrong one, funds rounding, the CMCD event-timeline +# record and the application/cmcd batch, a ledger entry's audit path) and +# tests/live_metering.rs (the committed corpus regenerates byte for byte from its fixed +# seeds and the reference relay's recorded record, wai-rs gives every vector its stated +# outcome, every reason of §9 refuses a vector, the large audience's relay record is the +# lesson's size, and §9 lists exactly the reference's codes); then the corpus's second +# implementation, live_metering_verify.py, written from the extension's text and sharing +# no code with wai-rs, which verifies each room with live_rooms_verify.py, each take with +# capture_take_verify.py and each composition receipt with live_compose_verify.py, +# recomputes every unit, total and grant's consumption itself, and requires every +# mutated copy of the corpus to fail; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-metering.md: tests/live_metering.rs requires this extension's §9 to list the +# codes the reference reports, live_metering_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_metering::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_metering) +python3 -I live-conformance/live_metering_verify.py live-conformance/metering diff --git a/.github/wai-conformance/verifiers/live-moderation.sh b/.github/wai-conformance/verifiers/live-moderation.sh new file mode 100644 index 000000000..c11dbb285 --- /dev/null +++ b/.github/wai-conformance/verifiers/live-moderation.sh @@ -0,0 +1,27 @@ +# step: live moderation · wai-rs and live_moderation_verify.py +# order: 210 +# features: live_moderation +# paths: wai/extensions/live-moderation.md +# +# wai.live.moderation (extensions/live-moderation.md): the reference module's unit tests +# (policies, statements and notices round-trip and their rules refuse, the export's +# strings and dates, the content types, the states that show an item) and +# tests/live_moderation.rs (every file of the committed corpus, vectors.json and each +# statement-of-reasons export, regenerates byte for byte from its fixed seeds and no +# other file is there, wai-rs gives every room, export and clip its stated outcome, and +# §10 lists exactly the reference's codes, each of which the corpus gives); then the +# corpus's second implementation, live_moderation_verify.py, written from the +# extension's text and sharing no code with wai-rs, which verifies each room with +# live_rooms_verify.py and each clip with live_record_verify.py, writes every export +# byte for byte, reads the extension's category, content-type and reason tables, and +# requires every mutated copy of the corpus to fail; standard library only, run +# isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-moderation.md: tests/live_moderation.rs requires this extension's §10 to +# list the codes the reference reports, live_moderation_verify.py reads its §9.5, §9.6 and §10 +# tables, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_moderation::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_moderation) +python3 -I live-conformance/live_moderation_verify.py live-conformance/moderation diff --git a/.github/wai-conformance/verifiers/live-provenance.sh b/.github/wai-conformance/verifiers/live-provenance.sh new file mode 100644 index 000000000..32cbcf98e --- /dev/null +++ b/.github/wai-conformance/verifiers/live-provenance.sh @@ -0,0 +1,27 @@ +# step: live provenance · wai-rs and c2pa_live_verify.py +# order: 280 +# features: live_provenance +# paths: wai/extensions/live-provenance.md +# +# wai.live.provenance (extensions/live-provenance.md): the reference module's unit tests +# (boxes, chunks, emsg and the BMFF hash; the JUMBF store; COSE_Sign1 and the OKP key; +# certificates written and read, their profile and chain; identifiers, track renderings, +# dates and the compositor's source type) and tests/live_provenance.rs (every file of the +# committed corpus regenerates byte for byte from its fixed seeds and the packaging +# corpus's AV1 floor, and no other file is there; wai-rs gives every vector its stated +# outcome; every reason of §12 refuses a vector; §12 lists exactly the reference's codes +# with the two C2PA codes it reports); then the corpus's second implementation, +# c2pa_live_verify.py, written from the extension's text and sharing no code with +# wai-rs, which reads CBOR, ISO BMFF, JUMBF, COSE and X.509 DER itself, verifies every +# signature with WAI's strict Ed25519 rule and every receipt with BLAKE3 from +# claim-conformance, reads the extension's reason table, and requires every mutated copy +# of a good vector to be refused; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-provenance.md: tests/live_provenance.rs requires this extension's §12 to +# list the codes and the C2PA codes the reference reports, c2pa_live_verify.py reads that table, +# and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_provenance::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_provenance) +python3 -I live-conformance/c2pa_live_verify.py live-conformance/c2pa-live diff --git a/.github/wai-conformance/verifiers/live-recording.sh b/.github/wai-conformance/verifiers/live-recording.sh new file mode 100644 index 000000000..f2132c4f1 --- /dev/null +++ b/.github/wai-conformance/verifiers/live-recording.sh @@ -0,0 +1,24 @@ +# step: live recording · wai-rs and live_record_verify.py +# order: 140 +# features: live_record +# paths: wai/extensions/live-recording.md +# +# wai.live.recording (extensions/live-recording.md): the reference module's unit tests, +# the output tree's (whose range proofs it adds, tested against RFC 9162 §2.1.3.2's +# check), and tests/live_record.rs (the committed corpus regenerates byte for byte from +# its fixed seeds, wai-rs gives every recording and clip its stated outcome, every reason +# of §9 appears in it, every take it says it reuses is capture binding's byte for byte, +# and §9 lists exactly the reference's codes); then the corpus's second implementation, +# live_record_verify.py, written from the extension's text and sharing no code with +# wai-rs, which checks range proofs by the extension's recursion and by RFC 9162 +# §2.1.3.2's own algorithm, verifies each take with capture_take_verify.py and every +# signature with WAI's strict Ed25519 (claim-conformance/_ed25519.py), and requires every +# mutated copy of the corpus to fail; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-recording.md: tests/live_record.rs requires this extension's §9 to list the +# codes the reference reports, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- live_record:: output_tree::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_record) +python3 -I live-conformance/live_record_verify.py live-conformance/recordings diff --git a/.github/wai-conformance/verifiers/live-rooms.sh b/.github/wai-conformance/verifiers/live-rooms.sh new file mode 100644 index 000000000..751cb970b --- /dev/null +++ b/.github/wai-conformance/verifiers/live-rooms.sh @@ -0,0 +1,21 @@ +# step: live rooms · wai-rs and live_rooms_verify.py +# order: 110 +# features: live_room +# paths: wai/extensions/live-rooms.md +# +# wai.live.room (extensions/live-rooms.md): the reference module's unit tests and +# tests/live_room.rs (the committed corpus regenerates byte for byte from its fixed +# seeds, wai-rs gives every vector its stated outcome, every reason of §11 refuses a +# vector, and §11 lists exactly the reference's codes); then the corpus's second +# implementation, live_rooms_verify.py, written from the extension's text and sharing +# no code with wai-rs, which verifies every event with live_events_verify.py and every +# signature with WAI's strict Ed25519 (claim-conformance/_ed25519.py), standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-rooms.md: tests/live_room.rs requires this extension's §11 to list the codes +# the reference reports, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_room::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_room) +python3 -I live-conformance/live_rooms_verify.py live-conformance/rooms diff --git a/.github/wai-conformance/verifiers/live-worlds.sh b/.github/wai-conformance/verifiers/live-worlds.sh new file mode 100644 index 000000000..be243a24b --- /dev/null +++ b/.github/wai-conformance/verifiers/live-worlds.sh @@ -0,0 +1,22 @@ +# step: live worlds · wai-rs and live_worlds_verify.py +# order: 640 +# features: live_world +# paths: wai/extensions/live-worlds.md +# +# wai.live.world (extensions/live-worlds.md): the reference module's unit tests (the +# statement and its rules, the grant attribute) and tests/live_world.rs (vectors.json +# regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives +# every world its stated outcome, and §11 lists exactly the reference's codes, each of which +# a verifier raises the corpus gives); then the corpus's second implementation, +# live_worlds_verify.py, written from the extension's text and the sections it cites and +# sharing no code with wai-rs: the statement, each grant's signature and attributes, each +# receipt in its binary form under the owner's chain, each op's signature, owner, kind, +# Lamport rule and equivocation, and the session hash. Standard library only, run isolated. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/live-worlds.md: tests/live_world.rs requires this extension's §11 to list the +# codes the reference reports. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_world::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_world) +python3 -I live-conformance/live_worlds_verify.py live-conformance/worlds diff --git a/.github/wai-conformance/verifiers/loc.sh b/.github/wai-conformance/verifiers/loc.sh new file mode 100644 index 000000000..c6b826aa5 --- /dev/null +++ b/.github/wai-conformance/verifiers/loc.sh @@ -0,0 +1,10 @@ +# step: loc_verify.py +# order: 760 +# +# LOC carriage (extensions/loc-carriage.md): the second implementation of joule-moq's loc +# module (whose tests/loc_vectors.rs runs in joule-code-conformance), written from the text, +# with its own MOQT -22 property decoder and BLAKE3 (claim-conformance/_blake3.py): it reads +# every object back to its envelope, writes it again byte for byte, gives every refused +# object and envelope its code, and requires every mutated copy, and the corpus under a +# copy of itself with a check dropped, to fail. +python3 -I live-conformance/loc_verify.py live-conformance/loc diff --git a/.github/wai-conformance/verifiers/media-identification.sh b/.github/wai-conformance/verifiers/media-identification.sh new file mode 100644 index 000000000..ee1f9a226 --- /dev/null +++ b/.github/wai-conformance/verifiers/media-identification.sh @@ -0,0 +1,22 @@ +# step: media identification · wai-rs and identify_verify.py +# order: 740 +# features: identify +# alone: - +# paths: wai/extensions/media-identify.md +# +# Media identification (extensions/media-identify.md): wai-rs's unit tests (hostile bytes +# through every walk, what a file reads as its bytes) and tests/identify.rs (the corpus +# regenerates byte for byte with every result; the extension's tables; `wai identify`); then +# identify_verify.py, written from the text and sharing no code with wai-rs but +# format_probe_verify.py's probes, which gives every file its whole result, holds ext/ to +# what an independent prober read of it, and requires every mutated copy to fail. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/media-identify.md: tests/identify.rs requires media-identify §6 and §7 to list +# the reference's signatures, and identify_verify.py reads those tables and the capabilities of +# its §5.1. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib identify::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test identify) +(cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_identify_vectors" verify ../identify-conformance) +python3 -I identify-conformance/identify_verify.py identify-conformance diff --git a/.github/wai-conformance/verifiers/native-sink.sh b/.github/wai-conformance/verifiers/native-sink.sh new file mode 100644 index 000000000..68f053f39 --- /dev/null +++ b/.github/wai-conformance/verifiers/native-sink.sh @@ -0,0 +1,27 @@ +# step: native sink · wai-rs and native_sink_verify.py +# order: 460 +# features: native_sink +# paths: wai/extensions/native-sink.md +# +# The native sink (extensions/native-sink.md): the reference module's unit tests (dates, +# durations and attribute lists read as a player reads them, origins serialized as RFC 6454 +# does, names that state a content hash and names that do not, the acknowledgements +# rebuilt from hashes alone equal to stream-audit's reference, a bundle that verifies and +# one refused for another owner or channel) and tests/native_sink.rs (vectors.json +# regenerates byte for byte from its seeds and no other file is there, every call gets the +# answer it states, §8 lists exactly the reasons the corpus reaches, and every case run +# through the C interface's symbols gets the same answers); then the corpus's second +# implementation, native_sink_verify.py, written from the extension's text and sharing no +# code with wai-rs, which replays every case, reads playlist receipts and their chain with +# http_verify.py, holds every bundle to its own session field by field and verifies it with +# stream_audit_verify.py, requires every mutated copy of the corpus to fail, and drops each +# of its own checks in turn and requires the corpus to catch the weakened verifier; +# standard library only, run isolated (-I). The Swift package runs in wai-sink-swift.yml. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/native-sink.md: tests/native_sink.rs requires this extension's §8 to list exactly +# the reasons its corpus reaches, and native_sink_verify.py reads that table. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib native_sink::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test native_sink) +python3 -I live-conformance/native_sink_verify.py live-conformance/native-sink diff --git a/.github/wai-conformance/verifiers/opportunity-report.sh b/.github/wai-conformance/verifiers/opportunity-report.sh new file mode 100644 index 000000000..8991d3d0b --- /dev/null +++ b/.github/wai-conformance/verifiers/opportunity-report.sh @@ -0,0 +1,30 @@ +# step: opportunity report · wai-rs and opportunity_report_verify.py +# order: 420 +# features: opportunity_report +# paths: wai/extensions/opportunity-report.md +# +# wai.delivery.opportunity (extensions/opportunity-report.md): the reference module's unit +# tests (the SCTE 35 subset reader against the standard's sample and damaged sections, the +# reported statement's JSON and CSV forms and dates, the report's and the check's reading and +# rules, the ratios); tests/opportunity_report.rs (vectors.json regenerates byte for byte +# from its fixed seeds and no other file is there; wai-rs gives every vector its stated +# outcome; every reason of §13 and every reused code refuses a vector; §13 lists exactly the +# reference's codes; the worked example, a channel's day whose reported statement +# under-reports one owner's programme, and the check that shows it, recomputed from the +# public records alone); then the corpus's second implementation, opportunity_report_verify.py, +# written from the extension's text and sharing no code with wai-rs, which verifies rooms +# with live_ads_verify.py and contracts, runs and signed statements with +# revenue_share_verify.py, reads every report, check, section and reported statement itself, +# recomputes every break, attribution, line, citation and flag, requires every mutated copy +# of the corpus to fail, and drops each of its own checks in turn and requires the corpus to +# catch the weakened verifier; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/opportunity-report.md: tests/opportunity_report.rs requires this extension's §13 +# to list the codes the reference reports, opportunity_report_verify.py reads that table, and +# the registry's wire identifiers test reads its sections. Its corpus is generated from fixed +# seeds, under the paths above. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib opportunity_report::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test opportunity_report) +python3 -I live-conformance/opportunity_report_verify.py live-conformance/opportunity diff --git a/.github/wai-conformance/verifiers/opus-dred.sh b/.github/wai-conformance/verifiers/opus-dred.sh new file mode 100644 index 000000000..679c9f153 --- /dev/null +++ b/.github/wai-conformance/verifiers/opus-dred.sh @@ -0,0 +1,35 @@ +# step: opus deep redundancy · wai-rs and dred_verify.py +# order: 220 +# features: dred_exact +# release-features: dred_exact +# paths: wai/extensions/opus-dred.md +# +# wai.audio.opus.dred.int (extensions/opus-dred.md): first the draft's weights file and +# test vectors, which the corpus names by SHA-256 and does not hold, fetched from the +# draft's locations and checked against its digests (live-conformance/dred/fetch.sh). +# Then the reference module's unit tests (the range decoder against its encoder, the +# extension draft's worked examples, payloads round-trip, weights quantized from their +# bits, the activation tables, the driver on a synthetic set), tests/dred_exact.rs (the +# corpus regenerates byte for byte from its seeds and the weights and no other file is +# there, every payload and case reaches its outcome, §8 lists exactly the module's codes, +# §4 states its digests), and in release tests/dred_exact_measure.rs (every record of the +# draft's test vectors decoded through the profile and through the floating-point +# network: the profile passes the draft's feature test, and §6's table is what it +# measures). Then the corpus's second implementation, dred_verify.py, written from the +# extension's text and sharing no code with wai-rs, which recomputes every outcome, +# digest and row in its own integer arithmetic and requires every mutated copy of the +# corpus to fail; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/opus-dred.md: tests/dred_exact.rs requires this extension's §8 to list the codes +# the reference reports and its §4 to state the reference's digests, tests/dred_exact_measure.rs +# its §6 table, dred_verify.py reads §4 and §8, and the registry's tests read SPEC §5 against +# it. +set -euo pipefail +d="${RUNNER_TEMP:-/tmp}/dred" +bash live-conformance/dred/fetch.sh "$d" vectors +w="$d/dred_unquantized_weights.bin" +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib dred_exact::) +(cd wai-rs && WAI_DRED_WEIGHTS="$w" cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test dred_exact) +(cd wai-rs && WAI_DRED_WEIGHTS="$w" WAI_DRED_VECTORS="$d" cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test dred_exact_measure) +python3 -I live-conformance/dred_verify.py live-conformance/dred "$w" diff --git a/.github/wai-conformance/verifiers/platform-encode.sh b/.github/wai-conformance/verifiers/platform-encode.sh new file mode 100644 index 000000000..83bc4ebce --- /dev/null +++ b/.github/wai-conformance/verifiers/platform-encode.sh @@ -0,0 +1,26 @@ +# step: platform-encode · wai-rs and encode_verify.py +# order: 570 +# features: platform_encode +# paths: wai/extensions/platform-encode.md +# +# wai.video.encode (extensions/platform-encode.md): the reference module's unit tests (the +# Exp-Golomb reader, emulation prevention, NAL splitting, length-prefixed samples, the +# resampler, the source reader, the strict JSON scan and the date-time) and +# tests/platform_encode.rs (the fixture streams are the pinned bytes, every generated file +# regenerates byte for byte, every case gets its outcome, every record reason has a case, and +# §12 lists exactly the reference's reasons); then the corpus's second implementation, +# encode_verify.py, written from the extension's text and sharing no code with wai-rs or +# wai-pack, which reads every Annex B stream, its access units and parameter sets itself, +# verifies every record with WAI's strict Ed25519 rule and every object with BLAKE3 from +# claim-conformance, checks the encoded renditions against the base packaging golden, +# requires every mutated copy of the corpus to fail, and drops each of its own checks in turn +# and requires the corpus or its mutations to catch the weakened verifier; standard library +# only, run isolated (-I). The encoders themselves run on the macOS leg (codec-wrappers). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/platform-encode.md: tests/platform_encode.rs requires this extension's §12 to +# list the reasons the reference reports, and encode_verify.py reads that table. +set -euo pipefail +(cd wai-rs && cargo test --locked --no-default-features --features "$WAI_FEATURES" --lib platform_encode::) +(cd wai-rs && cargo test --locked --no-default-features --features "$WAI_FEATURES" --test platform_encode) +python3 -I encode-conformance/encode_verify.py encode-conformance diff --git a/.github/wai-conformance/verifiers/prior-forms.sh b/.github/wai-conformance/verifiers/prior-forms.sh new file mode 100644 index 000000000..7f3d9e38b --- /dev/null +++ b/.github/wai-conformance/verifiers/prior-forms.sh @@ -0,0 +1,24 @@ +# step: prior forms · wai-rs and prior_forms_verify.py +# order: 330 +# features: prior_wire prior_wire,neural_int +# paths: wai/extensions/prior-carriage.md +# +# Tensor files as prior forms (extensions/prior-carriage.md §9 to §14): the module's unit +# tests (each layout read strictly and written canonically, the arithmetic coder's integers +# round-tripping and a damaged payload caught, the profile's checks in their order, both +# bindings through every form) and tests/prior_wire.rs (the committed corpus regenerates +# byte for byte from its pinned native artifacts; §13 lists exactly the reference's codes; +# every corpus prior in every form loads to its native bytes, and the two trained WSR1 +# priors give the same upscaled picture as their native files); then the corpus's second +# implementation, prior_forms_verify.py, written from the extension's text and sharing no +# code with wai-rs, which also requires every byte-flipped copy of an accepted file to be +# caught; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/prior-carriage.md: tests/prior_wire.rs requires this extension's §13 to list the +# codes the reference reports, prior_forms_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- prior_wire:: strict_json:: codecs::prior_form) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test prior_wire) +python3 -I live-conformance/prior_forms_verify.py live-conformance/prior-forms diff --git a/.github/wai-conformance/verifiers/qoe-aggregation.sh b/.github/wai-conformance/verifiers/qoe-aggregation.sh new file mode 100644 index 000000000..45fd777a0 --- /dev/null +++ b/.github/wai-conformance/verifiers/qoe-aggregation.sh @@ -0,0 +1,32 @@ +# step: QoE aggregation · wai-rs and telemetry_aggregate_verify.py +# order: 600 +# features: telemetry_aggregate +# paths: wai/extensions/telemetry-aggregation.md +# +# wai.telemetry.aggregate (extensions/telemetry-aggregation.md): the reference module's unit +# tests (the fields against a wide-arithmetic-free product, the generators' orders, +# TurboSHAKE128 against RFC 9861, HPKE against RFC 9180 A.1.1, the FLP accepting a valid +# measurement and refusing an invalid one, delivery-telemetry's key classes) and +# tests/telemetry_aggregate.rs (vectors.json regenerates byte for byte from its fixed seeds and +# no other file is there, vdaf-22/ is the draft's test vectors as published and wai-rs passes +# every one, every receipt, cell and delivery-only report gets its stated outcome, every +# reason of §11 and every reused code refuses a vector, §11 lists exactly the reference's +# codes, delivery-telemetry §3.4 classes exactly the reference's keys); then the corpus's +# second implementation, telemetry_aggregate_verify.py, written from the extension's text and +# the drafts it cites and sharing no code with wai-rs, which evaluates Prio3 itself and passes +# the draft's vectors, opens every report with its own X25519, HKDF, AES-128-GCM and HPKE, +# re-runs every flow (both aggregators' verification, the aggregation job byte for byte, the +# batch buckets, every refusal, the collection), verifies every receipt and cell with WAI's +# strict Ed25519 rule from claim-conformance, reads the key classes and the reason table from +# the texts, requires every mutated copy of the corpus to fail, and drops each of its own +# checks in turn and requires the corpus to catch the weakened verifier; standard library +# only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/telemetry-aggregation.md: tests/telemetry_aggregate.rs requires this extension's +# §11 to list the codes the reference reports, telemetry_aggregate_verify.py reads that table, +# and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- telemetry_aggregate:: telemetry::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test telemetry_aggregate) +python3 -I live-conformance/telemetry_aggregate_verify.py live-conformance/aggregate diff --git a/.github/wai-conformance/verifiers/quantum-referee.sh b/.github/wai-conformance/verifiers/quantum-referee.sh new file mode 100644 index 000000000..8b4f6e191 --- /dev/null +++ b/.github/wai-conformance/verifiers/quantum-referee.sh @@ -0,0 +1,15 @@ +# step: quantum referee.py +# order: 700 +# working-directory: . +# +# The quantum corpus's second implementation (quantum-sim extension), integers only: +# it derives the circuit/1 phase table by its recurrence and the circuit2 table by an +# interval-certified procedure, and requires both literal tables and their digests, +# and the extension's Appendix B to state the same; +# rebuilds every file of both vector trees byte for byte and recomputes every manifest +# entry with its own reader, simulator, sampler and BLAKE3; refuses every reject file +# with its listed class and by the rule its name states; and requires dyadic_ladder +# to see every ±1 change of every P(k) under both tables. Standard library only, run +# isolated (-I), so no module from the working directory, the environment or the +# user's site-packages is imported. +python3 -I wai/quantum-conformance/tools/referee.py wai/quantum-conformance diff --git a/.github/wai-conformance/verifiers/receipt-ledger.sh b/.github/wai-conformance/verifiers/receipt-ledger.sh new file mode 100644 index 000000000..350e8cfd6 --- /dev/null +++ b/.github/wai-conformance/verifiers/receipt-ledger.sh @@ -0,0 +1,34 @@ +# step: receipt ledger · wai-rs and receipt_ledger_verify.py +# order: 400 +# features: receipt_ledger +# paths: wai/extensions/receipt-ledger.md +# +# wai.delivery.ledger (extensions/receipt-ledger.md): the reference module's unit tests and +# tests/receipt_ledger.rs (vectors.json regenerates byte for byte from its fixed seeds and +# from the sources it takes from the metering, settlement, ads, presentation, stitched and +# routes corpora and the packs of pack-conformance, and no other file is there; wai-rs gives every vector its +# stated outcome; every reason of §10 and every reused code refuses a vector, and so does +# each source kind's own extension; §10 lists exactly the reference's codes; under a tariff +# of version 3, jobs priced per operation and a chain that bills a job's work once, its +# output's receipt cited again by a re-run, with version 2 billing as before); then the +# corpus's second implementation, receipt_ledger_verify.py, written from the extension's +# text and sharing no code with wai-rs, which verifies each source with that corpus's own +# second implementation (live_metering_verify.py, settlement_verify.py, live_ads_verify.py +# with ad_presentation_verify.py, stitched_presentation_verify.py, pack_verify.py, and for a +# media job route_record_verify.py and _int_motion_receipt.py) and +# reports a stitched stream's statement as that verifier gives it, checks signatures with +# WAI's strict Ed25519 rule, recomputes every line, citation, energy row and report, +# requires every mutated copy of the corpus to fail, and drops each of its own checks in turn and requires +# the corpus to catch the weakened verifier; run isolated (-I), with the `cryptography` package +# installed above for the OER/2 reference verifier that checks a job's hop receipts. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/receipt-ledger.md: tests/receipt_ledger.rs requires this extension's §10 to list +# the codes the reference reports, receipt_ledger_verify.py reads that table, and the registry's +# wire identifiers test reads its sections. Its corpus is generated from the metering, +# settlement, ads, presentation and stitched corpora and the packs of pack-conformance, all +# under the paths above. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib receipt_ledger::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test receipt_ledger) +python3 -I live-conformance/receipt_ledger_verify.py live-conformance/ledger diff --git a/.github/wai-conformance/verifiers/remote-desktop.sh b/.github/wai-conformance/verifiers/remote-desktop.sh new file mode 100644 index 000000000..92d7a5848 --- /dev/null +++ b/.github/wai-conformance/verifiers/remote-desktop.sh @@ -0,0 +1,35 @@ +# step: remote desktop · wai-rs, remote_desktop_verify.py and the page's module +# order: 590 +# features: remote_desktop +# paths: wai/extensions/remote-desktop.md +# +# wai.live.remote (extensions/remote-desktop.md): the reference module's unit tests (every +# statement read back and refused when a byte is short or over, the screen statement's +# field rules, the host's rules of §5.3 in their order, the grant attributes, a latency's +# legs, sealed actions: a seal with no plaintext in it that opens under the host's key +# alone, the commitment and what it binds, a seal that does not fit its actions, rules 8 +# and 9 after the others) and the crate's HPKE against RFC 9180 A.1.1; tests/remote_desktop.rs +# (vectors.json regenerates byte for byte from its fixed seeds and no other file is there, +# wai-rs gives every room its stated outcome at the commitment level and at the opened +# level, and §12 lists exactly the reference's codes, host refusals, floor refusals and +# the reasons a seal is not opened, each of which the corpus gives); +# then the corpus's second implementation, remote_desktop_verify.py, written from the +# extension's text and sharing no code with wai-rs, which verifies each room with +# live_rooms_verify.py, every signature with WAI's strict Ed25519 rule and every seal with +# the aggregation verifier's HPKE, recomputes every outcome from the rules, the floor and +# every latency's legs at both levels, requires every mutated copy of the corpus to fail and +# drops each of its own checks in turn and requires the corpus to catch the weakened +# verifier; standard library only, run isolated (-I). Last, the browser viewer's module +# (joule-moq-web/web/room/remote.js), which writes the corpus's input statements, floor +# statements and latency records byte for byte, seals every sealed input that opens again +# byte for byte, and gives every input the outcome the corpus states. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/remote-desktop.md: tests/remote_desktop.rs requires this extension's §12 to list +# the codes the reference reports, remote_desktop_verify.py reads its four tables, and the +# registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- remote_desktop:: hpke::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test remote_desktop) +python3 -I live-conformance/remote_desktop_verify.py live-conformance/remote +node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.test.mjs diff --git a/.github/wai-conformance/verifiers/revenue-share.sh b/.github/wai-conformance/verifiers/revenue-share.sh new file mode 100644 index 000000000..8cbbf1390 --- /dev/null +++ b/.github/wai-conformance/verifiers/revenue-share.sh @@ -0,0 +1,30 @@ +# step: revenue share · wai-rs and revenue_share_verify.py +# order: 410 +# features: revenue_share +# paths: wai/extensions/revenue-share.md +# +# wai.delivery.revenue_share (extensions/revenue-share.md): the reference module's unit tests +# (the largest-remainder division and its ties, the date-time a marker states, the contract's +# and the statement's reading and rules, contracts that overlap) and the channel receipt's +# JSON carriage; tests/revenue_share.rs (vectors.json regenerates byte for byte from its fixed +# seeds and the ledger corpus, and no other file is there; wai-rs gives every vector its +# stated outcome; every reason of §11 and every reused code refuses a vector; §11 lists exactly +# the reference's codes; the worked example, three statements of one month computed and then +# recomputed from the public records alone; the four operations from the command line); then +# the corpus's second implementation, revenue_share_verify.py, written from the extension's +# text and sharing no code with wai-rs, which verifies each ledger document with +# receipt_ledger_verify.py, reads every contract, run, marker and statement itself, +# recomputes every attribution, line, division and citation, requires every mutated copy of +# the corpus to fail, and drops each of its own checks in turn and requires the corpus to +# catch the weakened verifier; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/revenue-share.md: tests/revenue_share.rs requires this extension's §11 to list +# the codes the reference reports, revenue_share_verify.py reads that table, and the registry's +# wire identifiers test reads its sections. Its corpus is generated from the ledger corpus, +# under the paths above. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib revenue_share::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib wai_video_channel::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test revenue_share) +python3 -I live-conformance/revenue_share_verify.py live-conformance/share diff --git a/.github/wai-conformance/verifiers/robustness.sh b/.github/wai-conformance/verifiers/robustness.sh new file mode 100644 index 000000000..ade4fc600 --- /dev/null +++ b/.github/wai-conformance/verifiers/robustness.sh @@ -0,0 +1,30 @@ +# step: robustness evidence · wai-rs and robustness_verify.py +# order: 320 +# features: robustness +# release-features: robustness +# paths: wai/extensions/robustness-evidence.md +# +# Robustness evidence for soft bindings (extensions/robustness-evidence.md): the +# reference's unit tests (gamma is the rounded power, rates are exact and reduced, every +# attack of synthesis §7.5.7's battery is a transform that gives its bytes, parameters +# outside their ranges are refused), then tests/robustness.rs in release, since +# regenerating the corpus runs the harness over the measurement sets of synthesis §7.5.8 +# (every file of live-conformance/robustness/ regenerates byte for byte and no other file +# is there; every case gets what it states; §12 lists exactly the reference's codes and +# the corpus refuses a case for each; §4 registers exactly the reference's transforms; +# the tent64 records' tables are the synthesis corpus's battery.json; the tent64 claim is +# synthesis §7.5.8's list; jwp-receipts §3.1 lists the span reasons); then the second +# implementation, robustness_verify.py with _wai_mark.py, written from the extension's +# text and sharing no code with wai-rs, which reruns the conformance record's battery end +# to end and requires the same table, checks the tent64 records against battery.json and +# the claim against synthesis §7.5.8, mutates copies of the corpus and weakens its own +# verifier in turn; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/robustness-evidence.md: tests/robustness.rs requires this extension's §12 to list +# the codes the reference reports and its §4 to register the reference's transforms, +# robustness_verify.py reads both, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib robustness::) +(cd wai-rs && cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test robustness) +python3 -I live-conformance/robustness_verify.py live-conformance/robustness diff --git a/.github/wai-conformance/verifiers/route-record.sh b/.github/wai-conformance/verifiers/route-record.sh new file mode 100644 index 000000000..42e744451 --- /dev/null +++ b/.github/wai-conformance/verifiers/route-record.sh @@ -0,0 +1,30 @@ +# step: route record · wai-rs and route_record_verify.py +# order: 450 +# features: route_record +# paths: wai/extensions/route-record.md +# +# wai.route.record (extensions/route-record.md): the reference module's unit tests (a +# document read strictly and held to the subset, §2's encodings, an estimate past the subset +# unknown, a null ranked last, the expected figures per successful job and when they are +# unknown, the outputs' commitment, §8 listing exactly the reference's codes) and +# tests/route_record.rs (vectors.json regenerates byte for byte from its seeds and no other +# file is there; wai-rs gives every vector its stated outcome; every reason of §8 and every +# reused code refuses a vector; the accepted vectors are §10's, of version 1 and of version 2, +# every limit removing a candidate; a record is bound to each of its documents; `wai route +# verify` prints the outcome and exits by it); then the corpus's second implementation, +# route_record_verify.py, written from the extension's text and sharing no code with wai-rs, +# which checks the router's signature with WAI's strict Ed25519 rule and the hop receipts +# with the OER/2 reference verifier (the `cryptography` package, installed above), computes +# the outputs' commitment with claim-conformance/_blake3.py, recomputes every candidate, +# removal, expected figure, rank, order, decision and attempt, requires every mutated copy +# of the corpus to fail, and drops each of its own checks in turn and requires the corpus to +# catch the weakened verifier; run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/route-record.md: tests/route_record.rs and the module's tests require this +# extension's §8 to list the codes the reference reports, route_record_verify.py reads that +# table, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib route_record::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test route_record) +python3 -I live-conformance/route_record_verify.py live-conformance/routes diff --git a/.github/wai-conformance/verifiers/scene-carriage.sh b/.github/wai-conformance/verifiers/scene-carriage.sh new file mode 100644 index 000000000..b76097fc4 --- /dev/null +++ b/.github/wai-conformance/verifiers/scene-carriage.sh @@ -0,0 +1,23 @@ +# step: scene carriage · wai-rs and scene_verify.py +# order: 340 +# features: scene_map +# paths: wai/extensions/scene-carriage.md +# +# Scene carriage (extensions/scene-carriage.md): the module's unit tests (the conversions +# and spellings of §3 against fixed values, the containers an import builds equal to the +# ones wai-rs's world and audio-scene modules write, base64 and the USD subset's reader) +# and tests/scene_map.rs (the committed corpus regenerates byte for byte: every bundle +# written as glTF and USD and read back, every refusal reached; §8 lists exactly the +# reference's codes); then the corpus's second implementation, scene_verify.py, written +# from the extension's text and sharing no code with wai-rs, which also requires every +# byte-flipped copy of an exported document to be caught; standard library only, run +# isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/scene-carriage.md: tests/scene_map.rs requires this extension's §8 to list the +# codes the reference reports, scene_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- scene_map:: strict_json::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test scene_map) +python3 -I live-conformance/scene_verify.py live-conformance/scene diff --git a/.github/wai-conformance/verifiers/settlement.sh b/.github/wai-conformance/verifiers/settlement.sh new file mode 100644 index 000000000..5aae0dd73 --- /dev/null +++ b/.github/wai-conformance/verifiers/settlement.sh @@ -0,0 +1,29 @@ +# step: delivery settlement · wai-rs and settlement_verify.py +# order: 390 +# features: settlement +# paths: wai/extensions/settlement.md +# +# wai.delivery.settlement (extensions/settlement.md): the reference module's unit tests +# (range proofs over every range around every chunk boundary of objects of many lengths +# against BLAKE3's own root, tampered proofs refused, the group tree of jwp-receipts §2.2, +# records and item lists read and refused, amounts rounded down per line) and +# tests/settlement.rs (vectors.json regenerates byte for byte from its fixed seeds and no +# other file is there, wai-rs gives every statement and range proof its stated outcome, §11 +# lists exactly the reference's codes, each of which the corpus gives, beside every code it +# reuses); then the corpus's second implementation, settlement_verify.py, written from the +# extension's text and sharing no code with wai-rs, which verifies range proofs over +# BLAKE3's chunk tree with claim-conformance's BLAKE3, reads rooms, grants and proofs with +# live_rooms_verify.py and per-hop records with live_metering_verify.py, checks signatures +# with WAI's strict Ed25519 rule, recomputes every figure of every statement, reads the +# extension's reason table, requires every mutated copy of the corpus to fail, and drops +# each of its own checks in turn and requires the corpus to catch the weakened verifier; +# standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/settlement.md: tests/settlement.rs requires this extension's §11 to list the +# codes the reference reports, settlement_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib settlement::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test settlement) +python3 -I live-conformance/settlement_verify.py live-conformance/settlement diff --git a/.github/wai-conformance/verifiers/sim-diff.sh b/.github/wai-conformance/verifiers/sim-diff.sh new file mode 100644 index 000000000..1d6d8cb33 --- /dev/null +++ b/.github/wai-conformance/verifiers/sim-diff.sh @@ -0,0 +1,13 @@ +# step: sim_diff.py, wai_stage_measure against sim_verify.py +# order: 90 +# features: stage_measure +# +# The simulator differentially, outside the corpus: seeded random models (a refinement +# with its own delivery timeout among them) and plans, each simulated by +# wai_stage_measure sim and by sim_verify.py, the refusal or the whole result, the +# stall bound and the longest stall equal, and every stall within the bound where it +# applies. Two fixed seeds, so a run is reproducible. +set -euo pipefail +for seed in 1 2; do + python3 staged-measure-conformance/sim_diff.py "$RUNNER_TEMP/wai-bins/union/wai_stage_measure" "$seed" 1500 +done diff --git a/.github/wai-conformance/verifiers/sink-energy.sh b/.github/wai-conformance/verifiers/sink-energy.sh new file mode 100644 index 000000000..a1292164a --- /dev/null +++ b/.github/wai-conformance/verifiers/sink-energy.sh @@ -0,0 +1,31 @@ +# step: sink energy · wai-rs and sink_energy_verify.py +# order: 200 +# features: sink_energy +# paths: wai/extensions/sink-energy.md +# +# wai.sink.energy (extensions/sink-energy.md): the reference module's unit tests (a +# bracket summary is its intervals in integers, a round's figure and least declaration +# are energy-measurement §6.2's — checked against energy_report's reference — a round is +# refused by its first broken rule, a row seals its median and covers its spread, a +# segment's bounded window has no count term, the estimators and shares round half up, +# records sign and their identities cover the signature, §8 lists exactly the codes), the +# readers' unit tests (a counter is live only as a live rail advances, an advance crosses +# the wrap; on macOS, the per-process counter reads this process) and +# tests/sink_energy.rs (the committed corpus regenerates byte for byte from its fixed +# seeds, every case gets its stated outcome, every reason of §8 refuses a case, and every +# measured table under tables/ is accepted with its readings and procedure); then the +# corpus's second implementation, sink_energy_verify.py, written from the extension's text +# and sharing no code with wai-rs, which recomputes every round, row, segment and split in +# integers and requires every mutated copy of the corpus to fail; standard library only, +# run isolated (-I). These are the integer checks; the counters themselves are read by +# energy-hardware.yml's sink jobs, on the runners that have them. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/sink-energy.md: sink_energy's unit tests require this extension's §8 to list +# exactly the codes the reference reports, and sink_energy_verify.py reads that table. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib sink_energy::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib sink_meter::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test sink_energy) +(cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_sink_energy_vectors" verify ../live-conformance/sink-energy) +python3 -I live-conformance/sink_energy_verify.py live-conformance/sink-energy diff --git a/.github/wai-conformance/verifiers/staged-measure.sh b/.github/wai-conformance/verifiers/staged-measure.sh new file mode 100644 index 000000000..940eebb65 --- /dev/null +++ b/.github/wai-conformance/verifiers/staged-measure.sh @@ -0,0 +1,9 @@ +# step: measure_verify.py +# order: 70 +# +# The staged-measurement corpus's second implementation (staged-measure-conformance/README): +# it counts every object, applies every layer from its base's stage 0, computes the quality +# integers, the output tree, the prefixes, the overhead points and the dedup totals itself, +# and requires its decode to be the fixture tool's and its figures to be goldens.json's. +# Standard library only. +python3 staged-measure-conformance/measure_verify.py staged-measure-conformance diff --git a/.github/wai-conformance/verifiers/staged.sh b/.github/wai-conformance/verifiers/staged.sh new file mode 100644 index 000000000..e3ca85ae5 --- /dev/null +++ b/.github/wai-conformance/verifiers/staged.sh @@ -0,0 +1,9 @@ +# step: staged_verify.py +# order: 40 +# +# The staged-delivery corpus's second implementation (staged-conformance/README): it +# replays every case from its stage-0 buffers through its own reading of the +# extension (refines, WIR1, the sink's steps, holds, batches, conflicts, deadlines, +# tiers, stage hashes), decodes the WIV1 keyframe-0, PNG and FLAC bases, explodes the +# WAI2 files, and replays the schedule. Standard library only. +python3 staged-conformance/staged_verify.py staged-conformance diff --git a/.github/wai-conformance/verifiers/stitched-presentation.sh b/.github/wai-conformance/verifiers/stitched-presentation.sh new file mode 100644 index 000000000..be1282255 --- /dev/null +++ b/.github/wai-conformance/verifiers/stitched-presentation.sh @@ -0,0 +1,33 @@ +# step: stitched presentation · wai-rs, stitched_presentation_verify.py and the reference producer +# order: 260 +# features: stitched_presentation +# paths: wai/extensions/stitched-presentation.md +# +# wai.video.stitched_presentation (extensions/stitched-presentation.md): the reference +# module's unit tests (claims and pod-input records round-trip and their rules refuse, +# markers' bodies, a CPM price scaled to one impression, the XML subset) and the +# live-compose ad frames' (a squeeze's window and L tile the canvas, a double box's +# boxes), and tests/stitched_presentation.rs (every file of the committed corpus +# regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives +# every record, claim and ad its stated facts and every vector its spend by currency, and +# §9 lists exactly the codes the reference adds); then the tool's own check of the corpus +# through its binary; then the corpus's second implementation, +# stitched_presentation_verify.py, written from the extension's text and sharing no code +# with wai-rs, which reads every playlist, asset list, MPD, VAST document, form-1 receipt, +# record and claim itself, requires every mutated copy of the corpus to fail and catches +# each of its own checks dropped; standard library only, run isolated (-I). Last, the +# browser client's stitched producer (joule-moq-web/web/ads/stitched.js), which reads the +# markers from the corpus's manifests, measures an ad's region, and writes and signs the +# corpus's claims, byte for byte. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/stitched-presentation.md: tests/stitched_presentation.rs requires this +# extension's §9 to list the codes the reference adds, stitched_presentation_verify.py reads +# that table, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib stitched_presentation::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_compose::layout::frame_tests) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test stitched_presentation) +(cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_stitched_presentation" vectors verify ../live-conformance/stitched) +python3 -I live-conformance/stitched_presentation_verify.py live-conformance/stitched +node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/ads/stitched.test.mjs diff --git a/.github/wai-conformance/verifiers/stream-audit.sh b/.github/wai-conformance/verifiers/stream-audit.sh new file mode 100644 index 000000000..e02577c28 --- /dev/null +++ b/.github/wai-conformance/verifiers/stream-audit.sh @@ -0,0 +1,27 @@ +# step: stream audit · wai-rs and stream_audit_verify.py +# order: 430 +# features: stream_audit +# paths: wai/extensions/stream-audit.md +# +# wai.delivery.audit (extensions/stream-audit.md): the reference module's unit tests (the +# acknowledgement windows, chains and repeats of §4.2, the origin rule, the report's bytes, +# channel names) and tests/stream_audit.rs (vectors.json regenerates byte for byte from its +# fixed seeds and no other file is there, wai-rs gives every session and report its stated +# outcome, §14 lists exactly the reference's codes, each of which the corpus gives, beside +# every code it borrows); then the corpus's second implementation, stream_audit_verify.py, +# written from the extension's text and sharing no code with wai-rs, which reads +# acknowledgements with settlement_verify.py, claims with ad_presentation_verify.py and logs +# and energy classes with live_metering_verify.py, checks every signature with WAI's strict +# Ed25519 rule, recomputes every report, requires every mutated copy of the corpus to fail, +# and drops each of its own checks in turn and requires the corpus to catch the weakened +# verifier; standard library only, run isolated (-I). The collector and the player +# integration run in wai-audit.yml. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/stream-audit.md: tests/stream_audit.rs requires this extension's §14 to list the +# codes the reference reports, stream_audit_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib stream_audit::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test stream_audit) +python3 -I live-conformance/stream_audit_verify.py live-conformance/audit diff --git a/.github/wai-conformance/verifiers/stream-termination.sh b/.github/wai-conformance/verifiers/stream-termination.sh new file mode 100644 index 000000000..552dfbfbc --- /dev/null +++ b/.github/wai-conformance/verifiers/stream-termination.sh @@ -0,0 +1,30 @@ +# step: stream termination · wai-rs, wai-live-web and stream_termination_verify.py +# order: 630 +# features: stream_termination +# paths: wai/extensions/stream-termination.md +# +# wai.live.termination (extensions/stream-termination.md) and the display match of +# capability-constraints §4: the reference module's unit tests (sides without orientation, +# each exceedance alone, the least rendition alone where none is within, a writer that +# refuses after its termination) and tests/stream_termination.rs (the committed corpus +# regenerates byte for byte from its fixed seeds, wai-rs gives every match and log its +# stated outcome and facts, every reason refuses a log, §7 lists exactly the reference's +# codes and capability-constraints §4.2 its match bits); wai-live-web's functions for a +# page held to the same corpus natively, a page's log written through them verified; then +# the corpus's second implementation, stream_termination_verify.py, written from the two +# extensions' text and sharing no code with wai-rs, which recomputes every match and every +# log's outcome and facts and requires every mutated copy of the corpus to fail; standard +# library only, run isolated (-I). The browser's own run is joule-moq-web.yml's scenario +# termination. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/stream-termination.md: tests/stream_termination.rs requires this extension's §7 +# to list the codes the reference reports and capability-constraints §4.2 to name the match's +# bits, stream_termination_verify.py reads both, and the registry's wire identifiers test reads +# its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib stream_termination::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test stream_termination) +(cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_stream_termination_vectors" verify ../live-conformance/termination) +(cd wai-live-web && cargo test --locked --features int_video --lib termination_tests) +python3 -I live-conformance/stream_termination_verify.py live-conformance/termination diff --git a/.github/wai-conformance/verifiers/synthesis-mark.sh b/.github/wai-conformance/verifiers/synthesis-mark.sh new file mode 100644 index 000000000..b86f5ebfd --- /dev/null +++ b/.github/wai-conformance/verifiers/synthesis-mark.sh @@ -0,0 +1,34 @@ +# step: synthesis watermark · wai-rs and synthesis_mark_verify.py +# order: 300 +# features: synthesis_mark +# release-features: synthesis_mark +# paths: wai/extensions/synthesis.md +# +# The registered watermark schemes wai-mark:tent64 and wai-mark:tent64s +# (extensions/synthesis.md §7.4 to §7.9): the reference's unit tests (the code corrects 29 +# errors and its generator is the definition's, the layouts, the embedding and the +# detector in every orientation, the tiled detector's exact autocorrelation and its +# recovery from a crop and a rotation, the battery's resampling, rotation and keyframe), +# then tests/synthesis_mark.rs in release, since it embeds and attacks every picture of +# the measurement sets under both schemes (every file of live-conformance/synthesis-mark/ +# regenerates byte for byte, both measurements among them, and no other file is there; +# every embedding, detection and case gets what it states; the extension's definitions, +# generator polynomial and shuffles are the reference's; §7.5.8 and §7.9.7 claim only +# attacks the measurements show survived, and §7.5.6's and §7.9.5's figures are the +# measurements'); then the second implementation, synthesis_mark_verify.py with +# _wai_mark.py, written from the extension's text and sharing no code with wai-rs, which +# embeds, detects and decodes itself, both stages of the tiled detector among them, runs +# the battery's picture attacks, derives the frames the vectors name, verifies every case +# with synthesis_verify.py, mutates copies of the corpus and weakens its own verifier in +# turn; standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/synthesis.md: tests/synthesis.rs requires this extension's §12 to list the codes +# the reference reports and its §10.2 table to pin the reference's icons, synthesis_verify.py +# reads both, tests/synthesis_mark.rs and synthesis_mark_verify.py read its scheme's definition, +# generator, figures, table and claims (§7.4 to §7.5.8), and the registry's wire identifiers +# test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- synthesis::mark synthesis_mark::) +(cd wai-rs && cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test synthesis_mark) +python3 -I live-conformance/synthesis_mark_verify.py live-conformance/synthesis-mark diff --git a/.github/wai-conformance/verifiers/synthesis.sh b/.github/wai-conformance/verifiers/synthesis.sh new file mode 100644 index 000000000..d9ff64b87 --- /dev/null +++ b/.github/wai-conformance/verifiers/synthesis.sh @@ -0,0 +1,28 @@ +# step: synthesis provenance · wai-rs and synthesis_verify.py +# order: 290 +# features: synthesis +# paths: wai/extensions/synthesis.md +# +# wai.media.synthesis (extensions/synthesis.md): the reference module's unit tests (the +# pinned derivations by their rules, the class lattice, a generator's envelope) and +# tests/synthesis.rs (the committed corpus regenerates byte for byte from its fixed seeds +# and no other file is there, wai-rs gives every case its stated outcome and statements, +# §12 lists exactly the reference's codes, each of which the corpus gives, and §10.2's +# table pins exactly the reference's icons); then the corpus's second implementation, +# synthesis_verify.py, written from the extension's text and sharing no code with wai-rs, +# which runs the pinned derivations and writes each digital source type and C2PA +# statement itself, verifies each room with live_rooms_verify.py and every record with +# WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, reads the extension's icon +# and reason tables, and requires every mutated copy of the corpus to fail; standard +# library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/synthesis.md: tests/synthesis.rs requires this extension's §12 to list the codes +# the reference reports and its §10.2 table to pin the reference's icons, synthesis_verify.py +# reads both, tests/synthesis_mark.rs and synthesis_mark_verify.py read its scheme's definition, +# generator, figures, table and claims (§7.4 to §7.5.8), and the registry's wire identifiers +# test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib synthesis::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test synthesis) +python3 -I live-conformance/synthesis_verify.py live-conformance/synthesis diff --git a/.github/wai-conformance/verifiers/takedown.sh b/.github/wai-conformance/verifiers/takedown.sh new file mode 100644 index 000000000..b35cf1888 --- /dev/null +++ b/.github/wai-conformance/verifiers/takedown.sh @@ -0,0 +1,30 @@ +# step: takedown · wai-rs and takedown_verify.py +# order: 480 +# features: takedown +# paths: wai/extensions/takedown.md +# +# wai.media.takedown (extensions/takedown.md): the reference module's unit tests (times, +# clock identifiers, the published registry, the order of subjects, the lower median) and +# tests/takedown.rs (the committed corpus regenerates byte for byte from its fixed seeds +# and no other file is there, wai-rs gives every case its stated outcome and facts and +# every counts entry its rows, §11 lists exactly the reference's codes, each of which the +# corpus gives, and the registry the module carries is registry/takedown-clocks.json); +# then the corpus's second implementation, takedown_verify.py, written from the +# extension's text and sharing no code with wai-rs, which computes every closure, holder, +# purge time, clock verdict and counts row itself, reads derivation records and delivery +# receipts with edge_verify.py, origin claims with synthesis_verify.py, acknowledgements +# with settlement_verify.py and every record with WAI's strict Ed25519 rule and BLAKE3 +# from claim-conformance, requires every mutated copy of the corpus to fail, and drops +# each of its own checks in turn and requires the corpus to catch the weakened verifier; +# run isolated (-I). The reference relay's DVR store honours takedowns (§10.1), tested by +# joule-moq-relay's tests/dvr.rs in joule-code-conformance.yml. +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/takedown.md: tests/takedown.rs requires this extension's §11 to list the codes +# the reference reports, takedown_verify.py reads that table, and the registry's wire +# identifiers test reads its sections. The registry of clocks it reads is under wai/registry/ +# above. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib takedown::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test takedown) +python3 -I live-conformance/takedown_verify.py live-conformance/takedown diff --git a/.github/wai-conformance/verifiers/timed-text.sh b/.github/wai-conformance/verifiers/timed-text.sh new file mode 100644 index 000000000..907ddfc7b --- /dev/null +++ b/.github/wai-conformance/verifiers/timed-text.sh @@ -0,0 +1,24 @@ +# step: timed text · wai-rs and timed_text_verify.py +# order: 150 +# features: timed_text +# paths: wai/extensions/timed-text.md +# +# wai.text.timed (extensions/timed-text.md): the reference module's unit tests (cue model, +# annotation, WebVTT, IMSC1 and its render-model bound, timelines, disclosure, and the +# 608 encoder and decoder) and tests/timed_text.rs (every file of the committed corpus, +# vectors.json and each expected .vtt, .ttml and .608.hex, regenerates byte for byte +# from its fixed seeds and no other file is there, wai-rs gives every case its stated +# outcome, the corpus covers every 608 code path, and §12 lists exactly the reference's +# codes); then the corpus's second implementation, timed_text_verify.py, written from +# the extension's text and sharing no code with wai-rs, which verifies every event with +# live_events_verify.py, writes every output byte for byte from the text, parses each +# IMSC1 document with xml.etree, and requires every mutated copy of the corpus to fail; +# standard library only, run isolated (-I). +# +# The paths this step brings to the workflow's trigger, and why: +# wai/extensions/timed-text.md: tests/timed_text.rs requires this extension's §12 to list the +# codes the reference reports, and the registry's wire identifiers test reads its sections. +set -euo pipefail +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib timed_text::) +(cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test timed_text) +python3 -I live-conformance/timed_text_verify.py live-conformance/captions diff --git a/.github/workflows/repo-checks.yml b/.github/workflows/repo-checks.yml index 66e2ffa4e..7df9bf358 100644 --- a/.github/workflows/repo-checks.yml +++ b/.github/workflows/repo-checks.yml @@ -29,7 +29,13 @@ jobs: runs-on: linux-arm64 timeout-minutes: 30 steps: + # The whole history: the changelog check diffs a pull request against its base and + # reads, from the base's first-parent history, when the check reached main + # (scripts/check-changelog-fragment.py), and it fails rather than pass unchecked when + # it cannot. - uses: actions/checkout@v5 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: @@ -106,6 +112,34 @@ jobs: if: ${{ !cancelled() }} run: python scripts/check-wai-section-refs.py + # ── The lists wai-conformance.yml derives hold the hand lists they replaced ─ + # Its feature lists, test lists, verifier steps, corpora and gate paths are read from + # the files that own them (.github/wai-conformance/, the tags of wai/wai-rs/tests/). + - name: "workflow lists: the derivation fails closed (self-test)" + if: ${{ !cancelled() }} + run: bash scripts/wai-conformance-lists-selftest.sh + - name: "workflow lists: every derived list holds every entry of the hand list it replaced" + if: ${{ !cancelled() }} + run: python3 -I scripts/wai-conformance-lists.py selftest + + # ── A change to WAI carries its changelog entry, in a file of its own ────── + # wai/changes/README.md states the form; the entries go into wai/CHANGELOG.md at a + # release (scripts/changelog-assemble.py). A pull request opened before the check + # reached main passes as it is; the check reads when it was opened from the event + # payload, and when the check reached main from the base's first-parent history. + - name: "changelog: the check and the assembler fail closed (self-test)" + if: ${{ !cancelled() }} + run: bash scripts/changelog-fragments-selftest.sh + - name: "changelog: every entry file is one entry in the changelog's form" + if: ${{ !cancelled() }} + run: python3 -I scripts/changelog-assemble.py --check + - name: "changelog: a change to wai/ adds its entry (pull request)" + if: ${{ !cancelled() && github.event_name == 'pull_request' }} + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + python3 -I scripts/check-changelog-fragment.py \ + "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" + # ── Public text: no vanity metrics, no comparison, no company names ───── - name: "public text: no vanity metrics in committed Markdown" if: ${{ !cancelled() }} diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index 84fa1a73a..5d9fb631b 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -74,256 +74,29 @@ on: - 'wai/extensions/jwp-receipts.md' # The quantum referee requires this extension's Appendix B to state its phase tables. - 'wai/extensions/quantum-sim.md' - # tests/live_events.rs requires this extension's §9 to list the codes the reference - # reports, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/live-events.md' - # tests/live_room.rs requires this extension's §11 to list the codes the reference - # reports, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/live-rooms.md' - # tests/capture_take.rs requires this extension's §12 to list the codes the reference - # reports, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/capture-binding.md' - # tests/live_compose.rs requires this extension's §10 to list the codes the reference - # reports, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/live-compose.md' - # tests/live_record.rs requires this extension's §9 to list the codes the reference - # reports, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/live-recording.md' - # tests/timed_text.rs requires this extension's §12 to list the codes the reference - # reports, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/timed-text.md' - # tests/access_services.rs requires this extension's §10 to list the codes the reference - # reports, access_services_verify.py reads that table, and the registry's wire - # identifiers test reads its sections. - - 'wai/extensions/access-services.md' - # tests/live_metering.rs requires this extension's §9 to list the codes the reference - # reports, live_metering_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/live-metering.md' - # tests/energy_steering.rs requires this extension's §10 to list the codes the reference - # reports, energy_steering_verify.py reads that table, and the registry's wire - # identifiers test reads its sections. - - 'wai/extensions/energy-steering.md' - # tests/live_moderation.rs requires this extension's §10 to list the codes the reference - # reports, live_moderation_verify.py reads its §9.5, §9.6 and §10 tables, and the - # registry's wire identifiers test reads its sections. - - 'wai/extensions/live-moderation.md' - # tests/live_watermark.rs requires this extension's §11 to list the codes the reference - # reports, live_watermark_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/forensic-watermark.md' - # e2ee_verify.py reads this extension's §13 table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/confidentiality.md' - # tests/live_messaging.rs requires this extension's §12 to list the codes the reference - # reports, live_messaging_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/live-messaging.md' - # tests/live_ads.rs requires this extension's §10 to list the codes the reference - # reports, live_ads_verify.py reads that table, and the registry's wire identifiers test - # reads its sections. - - 'wai/extensions/live-ads.md' - # tests/ad_presentation.rs requires this extension's §11 to list the codes the reference - # reports, ad_presentation_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/ad-presentation.md' - # tests/key_release.rs requires this extension's §3.1, §3.2, §7.2 and §11 tables to be the - # reference's codes, key_release_verify.py reads those tables and §3.3, and the registry's - # wire identifiers test reads its sections. - - 'wai/extensions/key-release.md' - # tests/conform_result.rs requires this extension's §5 to list the codes the reference - # reports, conform_result_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/conformance-results.md' # The ad-presentation step runs the browser client's reference producer against the # corpus, and the stitched-presentation step runs its stitched producer. - 'joule-code/joule-code-rs/crates/joule-moq-web/web/ads/**' - # tests/stitched_presentation.rs requires this extension's §9 to list the codes the - # reference adds, stitched_presentation_verify.py reads that table, and the registry's - # wire identifiers test reads its sections. - - 'wai/extensions/stitched-presentation.md' - # tests/channel_lineup.rs requires this extension's §10 to list the codes the reference - # adds and runs its §9's commands as the text writes them, channel_lineup_verify.py reads - # its §10, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/channel-lineup.md' - # tests/live_provenance.rs requires this extension's §12 to list the codes and the C2PA - # codes the reference reports, c2pa_live_verify.py reads that table, and the registry's - # wire identifiers test reads its sections. - - 'wai/extensions/live-provenance.md' - # tests/derive_edge.rs requires this extension's §9 to list the codes the reference - # reports and its §4 the derivations, edge_verify.py reads both tables, and the - # registry's wire identifiers test reads its sections. - - 'wai/extensions/deriving-edge.md' - # tests/synthesis.rs requires this extension's §12 to list the codes the reference - # reports and its §10.2 table to pin the reference's icons, synthesis_verify.py reads - # both, tests/synthesis_mark.rs and synthesis_mark_verify.py read its scheme's - # definition, generator, figures, table and claims (§7.4 to §7.5.8), and the registry's - # wire identifiers test reads its sections. - - 'wai/extensions/synthesis.md' - # tests/exact_upscale.rs requires this extension's §8 to list the codes the reference - # reports, upscale_verify.py reads that table, and the registry's wire identifiers test - # reads its sections. - - 'wai/extensions/exact-upscale.md' - # tests/prior_wire.rs requires this extension's §13 to list the codes the reference - # reports, prior_forms_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/prior-carriage.md' - # tests/scene_map.rs requires this extension's §8 to list the codes the reference - # reports, scene_verify.py reads that table, and the registry's wire identifiers test - # reads its sections. - - 'wai/extensions/scene-carriage.md' - # tests/avatar_face.rs requires this extension's §10 to list the codes the reference - # reports and its §7.9 to state the training report's figures, avatar_face_verify.py - # reads §10, and the registry's tests read SPEC §5's registration of wai.avatar.face. - - 'wai/extensions/avatars.md' - # The tool that trained the corpus's generator and wrote its inputs and its report. + # The tool that trained the face corpus's generator and wrote its inputs and its report. - 'wai/tools/wai_avatar_face_train.py' - # tests/ai_usage.rs requires this extension's §9 to list the codes the reference reports, - # ai_usage_verify.py reads that table, and the registry's wire identifiers test reads its - # sections. - - 'wai/extensions/ai-usage.md' - # tests/contribution.rs requires this extension's §11 to list the codes the reference - # reports, contribution_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/contribution.md' - # live_contribution_verify.py reads this extension's §7 list of the link record's - # reasons and requires the corpus to raise each one. - - 'wai/extensions/live-contribution.md' - # tests/settlement.rs requires this extension's §11 to list the codes the reference - # reports, settlement_verify.py reads that table, and the registry's wire identifiers test - # reads its sections. - - 'wai/extensions/settlement.md' - # tests/receipt_ledger.rs requires this extension's §10 to list the codes the reference - # reports, receipt_ledger_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. Its corpus is generated from the metering, settlement, ads, - # presentation and stitched corpora and the packs of pack-conformance, all under the paths - # above. - - 'wai/extensions/receipt-ledger.md' - # tests/revenue_share.rs requires this extension's §11 to list the codes the reference - # reports, revenue_share_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. Its corpus is generated from the ledger corpus, under the paths - # above. - - 'wai/extensions/revenue-share.md' - # tests/opportunity_report.rs requires this extension's §13 to list the codes the reference - # reports, opportunity_report_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. Its corpus is generated from fixed seeds, under the paths above. - - 'wai/extensions/opportunity-report.md' - # tests/stream_audit.rs requires this extension's §14 to list the codes the reference - # reports, stream_audit_verify.py reads that table, and the registry's wire identifiers test - # reads its sections. - - 'wai/extensions/stream-audit.md' - # tests/fleet.rs requires this extension's §10 to list the codes the reference reports, - # fleet_verify.py reads that table, and the registry's wire identifiers test reads its - # sections. Its corpus is generated from the metering corpus, under the paths above. - - 'wai/extensions/fleet.md' - # tests/route_record.rs and the module's tests require this extension's §8 to list the - # codes the reference reports, route_record_verify.py reads that table, and the registry's - # wire identifiers test reads its sections. - - 'wai/extensions/route-record.md' - # tests/native_sink.rs requires this extension's §8 to list exactly the reasons its - # corpus reaches, and native_sink_verify.py reads that table. - - 'wai/extensions/native-sink.md' - # tests/browser_sink.rs requires this extension's §11 to list the codes the reference - # reports, browser_sink_verify.py reads its §2, §4.1, §4.2 and §11 tables and - # capability-constraints' §3.1 register, and the registry's wire identifiers test reads - # its sections. - - 'wai/extensions/browser-sink.md' + # browser_sink_verify.py reads capability-constraints' §3.1 register, and + # tests/stream_termination.rs requires its §4.2 to name the match's bits. - 'wai/extensions/capability-constraints.md' - # sink_energy's unit tests require this extension's §8 to list exactly the codes the - # reference reports, and sink_energy_verify.py reads that table. - - 'wai/extensions/sink-energy.md' # The browser sink's functions for a page, held to the same corpus natively. - 'wai/wai-live-web/**' - # tests/takedown.rs requires this extension's §11 to list the codes the reference - # reports, takedown_verify.py reads that table, and the registry's wire identifiers test - # reads its sections. The registry of clocks it reads is under wai/registry/ above. - - 'wai/extensions/takedown.md' - # tests/co_performance.rs requires this extension's §10 to list the codes the reference - # reports, co_performance_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/co-performance.md' - # tests/age_assurance.rs requires this extension's §11 to list the codes the reference - # reports, age_assurance_verify.py reads that table, and the registry's wire identifiers - # test reads its sections. - - 'wai/extensions/age-assurance.md' - # tests/influence.rs requires this extension's §9 to list the codes the reference adds, - # influence_verify.py reads that table, and the registry's wire identifiers test reads its - # sections. - - 'wai/extensions/influence.md' - # tests/direction.rs requires this extension's §13 to list the codes the reference reports, - # direction_verify.py reads that table, and the registry's wire identifiers test reads its - # sections. - - 'wai/extensions/direction.md' - # tests/bearer_receipts.rs requires this extension's §10 to list the codes the reference - # reports, and bearer_verify.py reads that table. - - 'wai/extensions/bearer-receipts.md' - # tests/carousel.rs requires §9 to list the reference's codes and Appendix A to be its table - # of K', and carousel_verify.py reads both. - - 'wai/extensions/carousel.md' - # tests/holdings.rs requires §7 to list the reference's codes, and holdings_verify.py reads it. - - 'wai/extensions/holdings.md' - # tests/exploration.rs requires this extension's §13 to list the codes the reference - # reports and its §3.2 table to name the reference's descriptors, exploration_verify.py - # reads §13, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/exploration.md' - # tests/platform_encode.rs requires this extension's §12 to list the reasons the reference - # reports, and encode_verify.py reads that table. - - 'wai/extensions/platform-encode.md' - # tests/co_watch.rs requires this extension's §12 to list the codes the reference reports, - # co_watch_verify.py reads that table, and the registry's wire identifiers test reads its - # sections. - - 'wai/extensions/co-watch.md' - # tests/live_world.rs requires this extension's §11 to list the codes the reference reports. - - 'wai/extensions/live-worlds.md' # The co-watch step runs the browser client's co-watch module against the corpus. - 'joule-code/joule-code-rs/crates/joule-moq-web/web/room/cowatch*' - # tests/remote_desktop.rs requires this extension's §12 to list the codes the reference - # reports, remote_desktop_verify.py reads its four tables, and the registry's wire - # identifiers test reads its sections. - - 'wai/extensions/remote-desktop.md' # The remote desktop step runs the browser viewer's module against the corpus. - 'joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote*' - # tests/dred_exact.rs requires this extension's §8 to list the codes the reference - # reports and its §4 to state the reference's digests, tests/dred_exact_measure.rs its §6 - # table, dred_verify.py reads §4 and §8, and the registry's tests read SPEC §5 against it. - - 'wai/extensions/opus-dred.md' - # tests/telemetry_aggregate.rs requires this extension's §11 to list the codes the - # reference reports, telemetry_aggregate_verify.py reads that table, and the registry's - # wire identifiers test reads its sections. - - 'wai/extensions/telemetry-aggregation.md' # tests/telemetry_aggregate.rs requires §3.4's table to class the keys the reference # classes, and telemetry_aggregate_verify.py reads that table and §4.1 and §4.2's shards. - 'wai/extensions/delivery-telemetry.md' - # tests/exact_tonemap.rs requires this extension's §9 to list the codes the reference - # reports and its §6 to state the corpus's measurement, tonemap_verify.py reads §3.3's - # table, §5.4's constant and §9, and the registry's wire identifiers test reads its - # sections. - - 'wai/extensions/exact-tonemap.md' - # tests/robustness.rs requires this extension's §12 to list the codes the reference - # reports and its §4 to register the reference's transforms, robustness_verify.py reads - # both, and the registry's wire identifiers test reads its sections. - - 'wai/extensions/robustness-evidence.md' - # tests/splat_exact.rs requires this extension's §10 to list the codes the reference - # reports and the draft schema under extensions/gltf/ to name §3.2's members, - # splat_exact_verify.py reads §10, and the registry's wire identifiers test reads its - # sections. - - 'wai/extensions/splat-exact.md' - - 'wai/extensions/gltf/**' # splat_exact_verify.py decodes each payload with this tool's whm1/1 lowering and # interpreter. - 'wai/tools/wai_ops_check.py' - # tests/pq_signatures.rs requires this extension's §8 to list the codes the reference - # reports, pq_verify.py reads that table, and the registry's wire identifiers test reads - # its §2. - - 'wai/extensions/pq-signatures.md' - # tests/format_probe.rs requires format-probes §2.4 and §12 to list the reference's probes - # and codes, and projection.md §4 and §5 to state its codes; format_probe_verify.py reads - # format-probes' tables; and the registry's wire identifiers test reads projection.md, - # loc-carriage.md and delivery-telemetry.md (whose §6.6 cmsd_read_verify.py reads). - - 'wai/extensions/format-probes.md' - # tests/identify.rs requires media-identify §6 and §7 to list the reference's signatures, - # and identify_verify.py reads those tables and the capabilities of its §5.1. - - 'wai/extensions/media-identify.md' + # tests/format_probe.rs requires projection.md §4 and §5 to state its codes, and the + # registry's wire identifiers test reads projection.md, loc-carriage.md and + # delivery-telemetry.md (whose §6.6 cmsd_read_verify.py reads). - 'wai/extensions/projection.md' - 'wai/extensions/loc-carriage.md' - 'wai/extensions/delivery-telemetry.md' @@ -338,15 +111,19 @@ on: - 'wai/research/catalogue-prior.md' - 'wai/research/catalogue-prior/**' - 'wai/tools/wai_catalogue_prior_verify.py' - # tests/stream_termination.rs requires this extension's §7 to list the codes the reference - # reports and capability-constraints §4.2 to name the match's bits, - # stream_termination_verify.py reads both, and the registry's wire identifiers test reads - # its sections. - - 'wai/extensions/stream-termination.md' - 'wai/wai-web/**' # The audio provenance check below reads the corpora above with this tool. - 'wai/tools/wai_audio_provenance_check.py' - '.github/workflows/wai-conformance.yml' + # The steps of the independent verifiers, the corpora of cargo-classes, and the script + # that derives this workflow's lists from them (scripts/wai-conformance-lists.py). + - '.github/wai-conformance/**' + - 'scripts/wai-conformance-lists.py' + # Each verifier step names the extension texts its tests and verifier read, in its own + # file (`# paths:`), so a new step adds no line here. The forge cannot read those files, + # so it starts this workflow on any extension text, and the gate below (the `relevant` + # job) runs it only for the paths the files name and the ones listed above. + - 'wai/extensions/**' # No global `shell:` default. The Windows runner is a host executor whose # `bash` is the WSL stub (a different OS), so shell-specific steps below pick @@ -362,7 +139,8 @@ jobs: # workflow's `paths:` against the push, so a merge of main into a branch fires the # workflow whatever the pull request changes; every job here waits on this answer. # A push to main, a schedule, or anything the script cannot determine answers - # `true` (scripts/pr-touches-paths.py). + # `true` (scripts/pr-touches-paths.py). The paths are the list above, with + # `wai/extensions/**` replaced by the paths the verifier steps' files name (--wide). relevant: name: does this change touch it runs-on: linux @@ -377,7 +155,8 @@ jobs: set -euo pipefail r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-conformance.yml \ "${{ github.event.pull_request.base.sha }}" \ - "${{ github.event.pull_request.head.sha }}") + "${{ github.event.pull_request.head.sha }}" \ + --paths-from .github/wai-conformance/verifiers --wide 'wai/extensions/**') echo "this change touches wai-conformance: $r" echo "run=$r" >> "$GITHUB_OUTPUT" @@ -431,6 +210,16 @@ jobs: rustup toolchain install 1.99.1 --profile minimal rustup default 1.99.1 + # scripts/wai-conformance-lists.py reads the integer kernels' tests and the corpora below. + - name: python3 for the derived lists (unix) + if: matrix.os != 'windows' + shell: bash + run: | + command -v python3 >/dev/null 2>&1 && exit 0 + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo" + $SUDO apt-get update -qq + $SUDO apt-get install -y -qq python3 + - name: Build the cargo classes (pure Rust, no C codec chain) working-directory: wai/wai-rs # neural_int joins the pure-Rust set: the integer-synthesis decoder is @@ -585,12 +374,17 @@ jobs: # each to the integers it states. Each of these runs every kernel set the leg's # processor runs (int_kernels::equivalence_policies: the wide kernels, and NEON on # arm64, AVX2 with or without VNNI on x86_64); int-kernels-amd below runs AVX-512. + # The tests of both steps are the files of tests/ whose line `// wai-conformance:` names + # int-kernels (and, in the release step only, int-kernels-release), and the features are + # their required-features (Cargo.toml) with `i32_forms`: scripts/wai-conformance-lists.py + # reads both and prints the command before it runs it, so a new test tags itself and edits + # no line here (its `dry-run` prints the lists). - name: Fast integer kernels ≡ reference kernels (overflow checks on) working-directory: wai/wai-rs - run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure,legacy_int,i32_forms --lib --test int_kernels_vectors --test int_ops_conformance --test int_kernels_geometry --test int_synth_ops --test int_ops_differential --test int_splat_ops --test int_audio_ops --test int_hyper_ops --test int_mlicpp --test int_mlicpp_preflight --test int_mlicpp_ops --test int_world_ops --test int_world_memory --test jpegai_ops --test jpegai_nets_ops --test int_video_ops --test wir1_ops --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_set_cache --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test int_video --test int_video_halfpel --test int_video_keyframe --test int_world --test int_world2 --test jpegai_bitstream_calibration + run: python3 ../../scripts/wai-conformance-lists.py test-tagged int-kernels --extra-features i32_forms -- --no-default-features --lib - name: Fast integer kernels ≡ reference kernels (release, overflow checks off) working-directory: wai/wai-rs - run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure,legacy_int,i32_forms --lib --test int_kernels_vectors --test int_ops_conformance --test int_kernels_geometry --test int_synth_ops --test int_ops_differential --test int_splat_ops --test int_audio_ops --test int_hyper_ops --test int_mlicpp --test int_mlicpp_preflight --test int_mlicpp_ops --test int_world_ops --test int_world_memory --test jpegai_ops --test jpegai_nets_ops --test int_video_ops --test wir1_ops --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_set_cache --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test manifest_parse_memory --test manifest_reader_equivalence --test int_video --test int_video_halfpel --test int_video_keyframe --test int_world --test int_world2 --test jpegai_bitstream_calibration + run: python3 ../../scripts/wai-conformance-lists.py test-tagged int-kernels int-kernels-release --extra-features i32_forms -- --release --no-default-features --lib # The integer decode path builds under each of its features alone, so a sink can take # one medium without the others, and for wasm32 (a browser sink); `int_payload` arms a # medium only under its feature, and none needs `neural`. So does the staged-measurement @@ -779,39 +573,17 @@ jobs: rustup target add --toolchain 1.99.1 wasm32-unknown-unknown cargo check --lib --target wasm32-unknown-unknown --no-default-features --features key_discovery,http - # The corpus-verify loop is genuinely shell-specific (a helper that groups - # log output and propagates the first non-zero exit), so it ships once per - # shell family. The command it runs is identical; only the wrapper differs. - - name: Every cargo-class corpus reproduces its frozen hashes (unix) - if: matrix.os != 'windows' - shell: bash + # Every cargo-class corpus, verified by its own tool (` verify `), each in a + # log group; the first that fails stops the step. The corpora are the files of + # .github/wai-conformance/corpora/, one a corpus naming its tool and directory, so a new + # corpus adds a file and edits no line here. The features are those the tools require + # (Cargo.toml) with provenance and video_manifest. scripts/wai-conformance-lists.py runs + # them and prints each command; `corpora` lists them, and `dry-run` prints the features. + # One step for every leg: the script is the wrapper each shell family needed (on Windows + # it needs python3 on the PATH). + - name: Every cargo-class corpus reproduces its frozen hashes working-directory: wai/wai-rs - run: | - set -euo pipefail - F=world,audio_scene,haptic,splat4d,score,provenance,neural_int,film,quantum,quantum_ops,oer,receipt_claims,key_discovery,video_int,audio_int,splat_int,staged,stage_measure,telemetry,http,video_manifest,pq_signatures - v() { echo "::group::$1"; cargo run -q --no-default-features --features "$F" --bin "$2" -- verify "$3"; echo "::endgroup::"; } - v worlds wai_world_vectors ../world-conformance - v spatial-audio wai_audio_vectors ../audio-conformance - v haptics wai_haptic_vectors ../haptic-conformance - v state-feeds wai_feed_vectors ../feed-conformance - v avatars wai_avatar_vectors ../avatar-conformance - v volumetric wai_splat4d_vectors ../volumetric-conformance - v score wai_score_vectors ../score-conformance - v int-synth wai_int_synth_vectors ../neural-conformance - v film wai_film_vectors ../film-conformance - v quantum wai_quantum_vectors ../quantum-conformance - v quantum-ops wai_quantum_ops_vectors ../quantum-ops-conformance - v oer wai_oer_vectors ../oer-conformance - v jwp wai_jwp_vectors ../jwp-conformance - v claims wai_claim_vectors ../claim-conformance - v pin wai_pin_vectors ../pin-conformance - v signers wai_signer_vectors ../signer-conformance - v int-payload wai_int_payload_vectors ../int-payload-conformance - v staged wai_staged_vectors ../staged-conformance - v http wai_http_vectors ../http-conformance - v staged-measure wai_stage_measure ../staged-measure-conformance - v telemetry wai_telemetry ../telemetry-conformance - v pq-signatures wai_pq_vectors ../pq-conformance + run: python3 ../../scripts/wai-conformance-lists.py corpora --verify --extra-features provenance,video_manifest # The pin corpus's second implementation: Python standard library only, sharing no # code with wai-rs. One leg is enough — it checks digests and listing rules, which # do not depend on the architecture — and the macOS host has python3. @@ -853,40 +625,6 @@ jobs: -o "$RUNNER_TEMP/settle_host" host_main.c settle.c sha256.c tweetnacl.c -lm "$RUNNER_TEMP/settle_host" "$RUNNER_TEMP/firmware_vectors.json" 400 cmp "$RUNNER_TEMP/firmware_vectors.json" ../firmware_vectors.json - - name: Every cargo-class corpus reproduces its frozen hashes (Windows) - if: matrix.os == 'windows' - shell: pwsh - working-directory: wai/wai-rs - run: | - $F = "world,audio_scene,haptic,splat4d,score,provenance,neural_int,film,quantum,quantum_ops,oer,receipt_claims,key_discovery,video_int,audio_int,splat_int,staged,stage_measure,telemetry,http,video_manifest,pq_signatures" - function v($grp, $bin, $path) { - Write-Output "::group::$grp" - cargo run -q --no-default-features --features $F --bin $bin -- verify $path - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - Write-Output "::endgroup::" - } - v worlds wai_world_vectors ../world-conformance - v spatial-audio wai_audio_vectors ../audio-conformance - v haptics wai_haptic_vectors ../haptic-conformance - v state-feeds wai_feed_vectors ../feed-conformance - v avatars wai_avatar_vectors ../avatar-conformance - v volumetric wai_splat4d_vectors ../volumetric-conformance - v score wai_score_vectors ../score-conformance - v int-synth wai_int_synth_vectors ../neural-conformance - v film wai_film_vectors ../film-conformance - v quantum wai_quantum_vectors ../quantum-conformance - v quantum-ops wai_quantum_ops_vectors ../quantum-ops-conformance - v oer wai_oer_vectors ../oer-conformance - v jwp wai_jwp_vectors ../jwp-conformance - v claims wai_claim_vectors ../claim-conformance - v pin wai_pin_vectors ../pin-conformance - v signers wai_signer_vectors ../signer-conformance - v int-payload wai_int_payload_vectors ../int-payload-conformance - v staged wai_staged_vectors ../staged-conformance - v http wai_http_vectors ../http-conformance - v staged-measure wai_stage_measure ../staged-measure-conformance - v telemetry wai_telemetry ../telemetry-conformance - v pq-signatures wai_pq_vectors ../pq-conformance # wai-quantum and the quantum corpus under wasm32-wasip2, run by wasmtime. On wasm32 # usize is 32 bits, so arithmetic that cannot overflow on a 64-bit host can there: a @@ -944,8 +682,12 @@ jobs: # The sets this host's processor runs, declared here (or by the runner); the # library test the_kernel_sets_found_are_the_declared_ones fails on any other. export WAI_INT_ISAS_EXPECT="${WAI_INT_AMD_ISAS_EXPECT:-wide,avx2,avx2-vnni,avx512,avx512-vnni}" - F=neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure,legacy_int,i32_forms - T="--test int_kernels_vectors --test int_kernels_geometry --test int_synth_ops --test int_splat_ops --test int_audio_ops --test int_hyper_ops --test int_mlicpp_ops --test int_world_ops --test jpegai_ops --test jpegai_nets_ops --test int_video_ops --test wir1_ops --test int_ops_conformance" + # The features of cargo-classes' kernel steps, and the test files tagged + # `// wai-conformance: int-kernels-amd` (scripts/wai-conformance-lists.py). + F=$(python3 ../../scripts/wai-conformance-lists.py features int-kernels --extra-features i32_forms) + T=$(python3 ../../scripts/wai-conformance-lists.py tests int-kernels-amd) + echo "features $F" + echo "tests $T" for p in "" --release; do nice -n 10 cargo test --locked $p --no-default-features --features $F --lib $T done @@ -1204,6 +946,16 @@ jobs: # lists beside its directory, extension and generator. # Each exits non-zero on any failure or any difference, and the step fails with it. # One platform: what they check is the reading of the text, not the architecture. + # + # Each step is a file of .github/wai-conformance/verifiers/: its shell body, run in wai/ + # with bash -eo pipefail as a workflow step runs, and a header that names its features, + # the binaries it runs, the paths it reads and its place in the order. Every list this job + # used to keep by hand is derived from those files when it runs (scripts/wai-conformance- + # lists.py), so a new verifier adds one file and edits none of this job's lists (a step + # whose `# paths:` falls under none of the trigger's paths still needs a trigger line, + # which the script's `check` names). The + # script's `dry-run` prints every list and step this job runs; `run --only ` + # runs one step. independent-verifiers: needs: relevant if: needs.relevant.outputs.run == 'true' @@ -1213,9 +965,6 @@ jobs: # No debuginfo, for the reason given at cargo-classes: this job ran out of the VM's disk # mid-link beside two others (#550). CARGO_PROFILE_DEV_DEBUG: '0' - WAI_FEATURES: key_discovery,http,video_manifest,stage_measure,live_events,live_room,capture_take,live_compose,live_record,timed_text,access_services,live_metering,energy_steering,live_moderation,dred_exact,live_ads,live_provenance,synthesis,synthesis_mark,neural_int,exact_upscale,robustness,ai_usage,contribution,settlement,browser_sink,receipt_claims,takedown,influence,direction,co_performance,age_assurance,bearer_receipts,exploration,co_watch,telemetry_aggregate,exact_tonemap,splat_exact,live_watermark,derive_edge,sink_energy,key_release,stitched_presentation,prior_wire,scene_map,avatar_face,receipt_ledger,stream_audit,fleet,native_sink,platform_encode,remote_desktop,stream_termination,live_world,format_probe,identify,live_messaging,conform_result,revenue_share,opportunity_report,channel_lineup,route_record,carousel,holdings - WAI_PQ_FEATURES: synthesis,pq_signatures - WAI_RELEASE_FEATURES: dred_exact,synthesis_mark,robustness,avatar_face steps: - uses: actions/checkout@v5 @@ -1245,1606 +994,53 @@ jobs: shell: bash run: python -m pip install cryptography - # Every wai-rs step below runs with one of the three feature lists this job declares, - # so cargo builds each target once and every later `cargo test` finds it fresh. A step - # selects its tests exactly as before (`--lib ::` or `--test `, with its - # filters); only the feature list is shared. Each feature still compiles alone in the - # `features-alone` job. + # Every wai-rs step below runs with one of three feature lists, so cargo builds each target + # once and every later `cargo test` finds it fresh. A step selects its tests exactly as + # before (`--lib ::` or `--test `, with its filters); only the feature list + # is shared. Each feature still compiles alone in the `features-alone` job. The lists are + # the steps' files' (the first step below prints them): # - # WAI_FEATURES the union of every step's features but the hybrid signatures'. - # WAI_PQ_FEATURES the hybrid signatures step. `pq_signatures` changes what a build - # without it refuses (pq-signatures §6.3: such a build reads a - # hybrid member as an unknown one), so it stays out of the union - # and every other step keeps testing a build without it. - # WAI_RELEASE_FEATURES the steps that run their tests optimised (`--release`). + # WAI_FEATURES the union of every step's `features:` but the hybrid signatures'. + # WAI_PQ_FEATURES the steps of `build: pq`, the hybrid signatures. `pq_signatures` + # changes what a build without it refuses (pq-signatures §6.3: such + # a build reads a hybrid member as an unknown one), so it stays out + # of the union and every other step keeps testing a build without it. + # WAI_RELEASE_FEATURES the steps' `release-features:`, for the tests they run optimised. # # Builds of the library that differ in features, or in the dependencies' features, # write the same file, so each switch between two of them rebuilds it. `cargo build` # and `cargo test` are two such builds (the dev-dependencies turn on wai-ops' `oracle` - # and serde_json's `preserve_order`), and so are two feature lists. Hence the order: - # the corpus tools the steps run as programs first, as `cargo build` builds them, - # copied to $RUNNER_TEMP/wai-bins/union and $RUNNER_TEMP/wai-bins/pq (the steps run - # those copies); then the union's test build, last, which every step but the hybrid - # signatures' runs; the release build has a directory of its own. The hybrid - # signatures step builds its tests itself, the one other build of the test profile. + # and serde_json's `preserve_order`), and so are two feature lists. Hence the order of + # the build step's script: the corpus tools the steps run as programs first (every + # `wai-bins/union/` and `wai-bins/pq/` a step names, and its `bins:` and + # `pq-bins:`), as `cargo build` builds them, copied to $RUNNER_TEMP/wai-bins/union and + # $RUNNER_TEMP/wai-bins/pq (the steps run those copies); then the union's test build, + # last, of every `--test` a step runs with "$WAI_FEATURES", which every step but the + # hybrid signatures' runs; the release build, of every `--test` run with + # "$WAI_RELEASE_FEATURES", has a directory of its own. The hybrid signatures step builds + # its tests itself, the one other build of the test profile. # - # The last step's differentials drive wai_claim_vectors' and wai_jwp_vectors' + # The claims step's differentials drive wai_claim_vectors' and wai_jwp_vectors' # `verdicts` modes, and compare wai_signer_vectors' key-discovery verdicts with # signer_verify.py's. Both builds of the three are copied and both are held to the # Python verifiers: the union's, which has key_discovery and no pq_signatures, and the # hybrid signatures', which reads the hybrid members the other refuses. + - name: The steps' feature lists, from their files + shell: bash + run: python3 scripts/wai-conformance-lists.py env | tee -a "$GITHUB_ENV" - name: Build wai-rs once for every step below, and the corpus verifiers it ships working-directory: wai/wai-rs shell: bash run: | set -euo pipefail - mkdir -p "$RUNNER_TEMP/wai-bins/union" "$RUNNER_TEMP/wai-bins/pq" - cargo build --no-default-features --features "$WAI_FEATURES" \ - --bin wai_http_vectors --bin wai_stage_measure --bin wai_browser_sink_vectors \ - --bin wai_claim_vectors --bin wai_jwp_vectors --bin wai_signer_vectors \ - --bin wai_sink_energy_vectors --bin wai_stitched_presentation \ - --bin wai_stream_termination_vectors --bin wai_format_probe --bin wai_identify_vectors --bin wai_conform_result_vectors \ - --bin wai_channel_lineup_vectors - for b in wai_http_vectors wai_stage_measure wai_browser_sink_vectors \ - wai_claim_vectors wai_jwp_vectors wai_signer_vectors \ - wai_sink_energy_vectors wai_stitched_presentation \ - wai_stream_termination_vectors wai_format_probe wai_identify_vectors wai_conform_result_vectors \ - wai_channel_lineup_vectors; do - cp "${CARGO_TARGET_DIR:-target}/debug/$b" "$RUNNER_TEMP/wai-bins/union/" - done - # Copied before this build replaces target/debug's copies of the three it shares. - cargo build --no-default-features --features "$WAI_PQ_FEATURES" \ - --bin wai_pq_vectors --bin wai_claim_vectors --bin wai_jwp_vectors --bin wai_signer_vectors - for b in wai_pq_vectors wai_claim_vectors wai_jwp_vectors wai_signer_vectors; do - cp "${CARGO_TARGET_DIR:-target}/debug/$b" "$RUNNER_TEMP/wai-bins/pq/" - done - cargo test --no-run --no-default-features --features "$WAI_FEATURES" --lib \ - --test live_events --test live_room --test capture_take --test live_compose --test live_record \ - --test timed_text --test access_services --test live_metering --test energy_steering \ - --test live_moderation --test live_ads --test ad_presentation --test live_provenance \ - --test synthesis --test exact_upscale --test cli_derive --test ai_usage --test contribution --test settlement \ - --test browser_sink --test takedown --test influence --test direction --test co_performance \ - --test age_assurance --test bearer_receipts --test carousel --test holdings --test exploration --test co_watch \ - --test telemetry_aggregate --test exact_tonemap --test splat_exact --test live_watermark \ - --test derive_edge --test det_cost_bin --test sink_energy --test key_release \ - --test stitched_presentation --test prior_wire --test scene_map --test receipt_ledger \ - --test stream_audit --test fleet --test native_sink --test platform_encode --test remote_desktop \ - --test stream_termination --test live_world --test format_probe --test identify --test live_messaging \ - --test conform_result --test revenue_share --test opportunity_report --test channel_lineup --test route_record - cargo test --release --no-run --no-default-features --features "$WAI_RELEASE_FEATURES" \ - --test dred_exact --test dred_exact_measure --test synthesis_mark --test robustness \ - --test avatar_face - - # The integer payload corpus's second implementation (int-payload-conformance/README): - # it parses every payload with the extension's rules and check order, converts every - # JSON source independently and requires the committed bytes, decodes the WIV1 - # keyframe-0 payloads in full to their sidecars' SHA-256, and checks every refusal - # fixture. Standard library only, so it needs nothing installed. - - name: int_payload_verify.py - working-directory: wai - shell: bash - run: python3 int-payload-conformance/int_payload_verify.py int-payload-conformance - # The determinism-cost record (research/determinism-cost.md): its summary recomputed - # from its readings by a second implementation that shares no code with the tool that - # wrote it (tools/wai_determinism_cost_verify.py, standard library only, run isolated), - # and every figure the research page and determinism-tiers §7 quote from it checked - # against it (each is followed by a `det` marker naming its place in the record). Then - # wai_det_cost, the decoder the record ran, held to the committed int_hyper vector and - # every WIH1 refusal fixture of int-payload-conformance/. - - name: determinism cost · the record, the text quoting it, and wai_det_cost - working-directory: wai - shell: bash - run: | - set -euo pipefail - python3 -I tools/wai_determinism_cost_verify.py research/determinism-cost/record.json \ - research/determinism-cost.md extensions/determinism-tiers.md - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test det_cost_bin) - # The catalogue-prior record (research/catalogue-prior.md): each reading checked against - # the SHA-256 its run.json states; every BD-rate the run stored recomputed from the - # rate and quality points, and the rate proxy, the recurrence and the decision rule's - # verdicts from the readings, by a second implementation that shares no code with the - # scripts that took them (tools/wai_catalogue_prior_verify.py, standard library only, - # run isolated); and every figure the page quotes checked (each is followed by a `cat` - # marker naming its value). - - name: catalogue prior · the record and the text quoting it - working-directory: wai - shell: bash - run: python3 -I tools/wai_catalogue_prior_verify.py research/catalogue-prior research/catalogue-prior.md - # The staged-delivery corpus's second implementation (staged-conformance/README): it - # replays every case from its stage-0 buffers through its own reading of the - # extension (refines, WIR1, the sink's steps, holds, batches, conflicts, deadlines, - # tiers, stage hashes), decodes the WIV1 keyframe-0, PNG and FLAC bases, explodes the - # WAI2 files, and replays the schedule. Standard library only. - - name: staged_verify.py - working-directory: wai - shell: bash - run: python3 staged-conformance/staged_verify.py staged-conformance - # The HTTP delivery corpus's second implementation (http-conformance/README): the head, - # the head proof (every committed proof rebuilt from the text), response checks, plans, - # selection, the byte-range procedure and its fallback, the codecs union and floor, the - # playlists (read, rendered, held adjacent, the classical master unchanged), playlist - # receipts (WAI's strict Ed25519, _ed25519.py) and the live edge, and the adaptation policies, - # checked first against published BLAKE3 vectors, with the standard library alone; - # and the receipts, key logs, grants files and parameter sets of §8.1–§8.5, through - # http_receipts.py and the corpora's own verifiers of jwp-receipts, which need the - # `cryptography` package installed above (without it those cases fail, never skip). - # Its mutations and the differential against wai-rs run in wai-http.yml's falsify - # job. - - name: http_verify.py - working-directory: wai - shell: bash - run: python3 http-conformance/http_verify.py http-conformance - # Outside the corpus: wai_http_vectors fuzz writes seeded cases (Content-Range values, - # codecs values, #WAI-LAYER lines, manifests and their member types, encryption, - # refines, layers declared against their bases, layers over WAI2 components, objects - # past the first request through a tampering origin; playlists, adjacent pairs, - # published masters, renderings, adaptation states, the integer log and changed - # receipt files; receipt files, indexes, grants files and key logs of §8.1–§8.5, - # whose cases need the `cryptography` package installed above) with wai-rs's - # outcomes, and - # http_verify.py must give every one, a refusal by the same code. It prints each - # family's count of cases and of refusals. Two fixed seeds, so a run is reproducible. - - name: The HTTP differential outside the corpus (wai-rs against http_verify.py) - working-directory: wai - shell: bash - run: | - set -euo pipefail - for seed in 11 12; do - "$RUNNER_TEMP/wai-bins/union/wai_http_vectors" fuzz http-conformance --seed "$seed" --count 400 --out "$RUNNER_TEMP/http-fuzz-$seed.json" - python3 http-conformance/http_verify.py cases "$RUNNER_TEMP/http-fuzz-$seed.json" http-conformance - done - # The staged-measurement corpus's second implementation (staged-measure-conformance/README): - # it counts every object, applies every layer from its base's stage 0, computes the quality - # integers, the output tree, the prefixes, the overhead points and the dedup totals itself, - # and requires its decode to be the fixture tool's and its figures to be goldens.json's. - # Standard library only. - - name: measure_verify.py - working-directory: wai - shell: bash - run: python3 staged-measure-conformance/measure_verify.py staged-measure-conformance - # The delivery simulator's second implementation: Appendix B from the text, every case - # of staged-measure-conformance/sim/ reproduced and its claim checked, with a - # counter-case required for each part of the stall bound a schedule can lose. - - name: sim_verify.py - working-directory: wai - shell: bash - run: python3 staged-measure-conformance/sim_verify.py staged-measure-conformance - # The simulator differentially, outside the corpus: seeded random models (a refinement - # with its own delivery timeout among them) and plans, each simulated by - # wai_stage_measure sim and by sim_verify.py, the refusal or the whole result, the - # stall bound and the longest stall equal, and every stall within the bound where it - # applies. Two fixed seeds, so a run is reproducible. - - name: sim_diff.py, wai_stage_measure against sim_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - for seed in 1 2; do - python3 staged-measure-conformance/sim_diff.py "$RUNNER_TEMP/wai-bins/union/wai_stage_measure" "$seed" 1500 - done - # wai.live.events (extensions/live-events.md): the reference module's unit tests and - # tests/live_events.rs (the committed corpus regenerates byte for byte from its fixed - # seeds, wai-rs gives every vector, policy case and binding case its stated outcome, - # the corpus covers every kind and reason, and §9 lists exactly the reference's - # codes); then the corpus's second implementation, live_events_verify.py, written - # from the extension's text and sharing no code with wai-rs, which verifies the same - # cases with WAI's strict Ed25519 (claim-conformance/_ed25519.py) and BLAKE3 - # (claim-conformance/_blake3.py), standard library only, run isolated (-I). - - name: live events · wai-rs and live_events_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_events::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_events) - python3 -I live-conformance/live_events_verify.py live-conformance/events - # wai.live.room (extensions/live-rooms.md): the reference module's unit tests and - # tests/live_room.rs (the committed corpus regenerates byte for byte from its fixed - # seeds, wai-rs gives every vector its stated outcome, every reason of §11 refuses a - # vector, and §11 lists exactly the reference's codes); then the corpus's second - # implementation, live_rooms_verify.py, written from the extension's text and sharing - # no code with wai-rs, which verifies every event with live_events_verify.py and every - # signature with WAI's strict Ed25519 (claim-conformance/_ed25519.py), standard - # library only, run isolated (-I). - - name: live rooms · wai-rs and live_rooms_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_room::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_room) - python3 -I live-conformance/live_rooms_verify.py live-conformance/rooms - # wai.capture.take (extensions/capture-binding.md): the reference module's unit tests, - # the output tree's (whose RFC 6962 hash and RFC 9162 consistency proofs the take tree - # reuses), and tests/capture_take.rs (the committed corpus regenerates byte for byte - # from its fixed seeds, wai-rs gives every vector its stated outcome, every reason of - # §12 appears in it, and §12 lists exactly the reference's codes); then the corpus's - # second implementation, capture_take_verify.py, written from the extension's text and - # sharing no code with wai-rs, which verifies each room with live_rooms_verify.py and - # every signature with WAI's strict Ed25519 (claim-conformance/_ed25519.py), and - # requires every mutated copy of the corpus to fail; standard library only, run - # isolated (-I). - - name: capture take · wai-rs and capture_take_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- capture_take:: output_tree::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test capture_take) - python3 -I live-conformance/capture_take_verify.py live-conformance/takes - # wai.live.compose (extensions/live-compose.md): the reference module's unit tests (the - # layout over every grid of 1 to 64 tiles stays on its canvas with no tile overlapping - # another, the fill modes, the attributes, the mixer) and tests/live_compose.rs (the - # committed corpus regenerates byte for byte from its fixed seeds, wai-rs gives every - # case its stated outcome, every reason of §10 refuses a case, and §10 lists exactly - # the reference's codes); then the corpus's second implementation, - # live_compose_verify.py, written from the extension's text and sharing no code with - # wai-rs, which verifies each room with live_rooms_verify.py, steps through its - # lifecycle replay to each instant itself, gives every layout, mix, descriptor, - # composition and receipt the same outcome rectangle for rectangle and sample for - # sample, and requires every mutated copy of the corpus to fail; standard library - # only, run isolated (-I). - - name: live compose · wai-rs and live_compose_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_compose::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_compose) - python3 -I live-conformance/live_compose_verify.py live-conformance/compose - # wai.live.recording (extensions/live-recording.md): the reference module's unit tests, - # the output tree's (whose range proofs it adds, tested against RFC 9162 §2.1.3.2's - # check), and tests/live_record.rs (the committed corpus regenerates byte for byte from - # its fixed seeds, wai-rs gives every recording and clip its stated outcome, every reason - # of §9 appears in it, every take it says it reuses is capture binding's byte for byte, - # and §9 lists exactly the reference's codes); then the corpus's second implementation, - # live_record_verify.py, written from the extension's text and sharing no code with - # wai-rs, which checks range proofs by the extension's recursion and by RFC 9162 - # §2.1.3.2's own algorithm, verifies each take with capture_take_verify.py and every - # signature with WAI's strict Ed25519 (claim-conformance/_ed25519.py), and requires every - # mutated copy of the corpus to fail; standard library only, run isolated (-I). - - name: live recording · wai-rs and live_record_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- live_record:: output_tree::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_record) - python3 -I live-conformance/live_record_verify.py live-conformance/recordings - # wai.text.timed (extensions/timed-text.md): the reference module's unit tests (cue model, - # annotation, WebVTT, IMSC1 and its render-model bound, timelines, disclosure, and the - # 608 encoder and decoder) and tests/timed_text.rs (every file of the committed corpus, - # vectors.json and each expected .vtt, .ttml and .608.hex, regenerates byte for byte - # from its fixed seeds and no other file is there, wai-rs gives every case its stated - # outcome, the corpus covers every 608 code path, and §12 lists exactly the reference's - # codes); then the corpus's second implementation, timed_text_verify.py, written from - # the extension's text and sharing no code with wai-rs, which verifies every event with - # live_events_verify.py, writes every output byte for byte from the text, parses each - # IMSC1 document with xml.etree, and requires every mutated copy of the corpus to fail; - # standard library only, run isolated (-I). - - name: timed text · wai-rs and timed_text_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib timed_text::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test timed_text) - python3 -I live-conformance/timed_text_verify.py live-conformance/captions - # wai.access.services (extensions/access-services.md): the reference module's unit tests - # (words, full and prefix alignment with their walk back, signed integers, gains) and - # tests/access_services.rs (vectors.json and each DAPT document regenerate byte for byte - # from their fixed seeds and no other file is there, wai-rs gives every vector and - # document its stated outcome and figures, every reason of §10 refuses a case, and §10 - # lists exactly the reference's codes); then the corpus's second implementation, - # access_services_verify.py, written from the extension's text and sharing no code with - # wai-rs, which verifies events and captions with timed_text_verify.py and every record - # with WAI's strict Ed25519 rule, computes every figure and writes every DAPT document - # itself, parses each with xml.etree, reads the extension's reason table, and requires - # every mutated copy of the corpus to fail; standard library only, run isolated (-I). - - name: access services · wai-rs and access_services_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib access_services::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test access_services) - python3 -I live-conformance/access_services_verify.py live-conformance/access - # conform-result/1 (extensions/conformance-results.md): the reference module's unit tests - # (a signed record accepted and every member covered, the strict rule's small-order forgery - # refused, the date form, a vector for every reason) and tests/conform_result.rs (the - # committed corpus regenerates byte for byte, wai-rs gives every vector its stated outcome, - # every reason of §5 refuses a vector, §5 lists exactly the reference's codes, and no leaf - # of a record survives a change after signing); then wai_conform_result_vectors verify, and - # the corpus's second implementation, conform_result_verify.py, written from the - # extension's text and sharing no code with wai-rs (WAI's strict Ed25519 rule from - # claim-conformance/_ed25519.py), which reads the extension's reason table; and with each - # check of §4 left out in turn (--skip), the corpus must fail it. Standard library only, - # run isolated (-I). - - name: conformance results · wai-rs and conform_result_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib conform_result::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test conform_result) - "$RUNNER_TEMP/wai-bins/union/wai_conform_result_vectors" verify conform-result-conformance - python3 -I conform-result-conformance/conform_result_verify.py conform-result-conformance - for c in duplicates format members signature summary verdict; do - python3 -I conform-result-conformance/conform_result_verify.py --skip "$c" conform-result-conformance | tail -1 - done - # wai.live.metering (extensions/live-metering.md): the reference module's unit tests - # (records and ledgers round-trip, the energy rules, the size bound at its limit, a - # total of two classes told from a wrong one, funds rounding, the CMCD event-timeline - # record and the application/cmcd batch, a ledger entry's audit path) and - # tests/live_metering.rs (the committed corpus regenerates byte for byte from its fixed - # seeds and the reference relay's recorded record, wai-rs gives every vector its stated - # outcome, every reason of §9 refuses a vector, the large audience's relay record is the - # lesson's size, and §9 lists exactly the reference's codes); then the corpus's second - # implementation, live_metering_verify.py, written from the extension's text and sharing - # no code with wai-rs, which verifies each room with live_rooms_verify.py, each take with - # capture_take_verify.py and each composition receipt with live_compose_verify.py, - # recomputes every unit, total and grant's consumption itself, and requires every - # mutated copy of the corpus to fail; standard library only, run isolated (-I). - - name: live metering · wai-rs and live_metering_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_metering::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_metering) - python3 -I live-conformance/live_metering_verify.py live-conformance/metering - # wai.live.steering (extensions/energy-steering.md): the reference module's unit tests - # (policies and decisions round-trip and their rules refuse, the window, the request - # reader over HLS, DASH and CMCD, the RFC 9651 parser, exact products at their limits, - # the margin at its boundary, figures compared only within a class, and a pathway with - # no figure never placed first by its rank while one with a figure is within the - # limits) and tests/energy_steering.rs (the committed corpus regenerates byte for byte - # from its fixed seeds, wai-rs gives every request and vector its stated outcome, every - # reason of §10 refuses a vector, and §10 lists exactly the reference's codes); then the - # corpus's second implementation, energy_steering_verify.py, written from the - # extension's text and sharing no code with wai-rs, which reads the room with - # live_rooms_verify.py and the per-hop records with live_metering_verify.py, parses RFC - # 9651 itself, recomputes every report, figure, decision and manifest, and requires - # every mutated copy of the corpus to fail; standard library only, run isolated (-I). - - name: energy steering · wai-rs and energy_steering_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib energy_steering::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test energy_steering) - python3 -I live-conformance/energy_steering_verify.py live-conformance/steering - # wai.sink.energy (extensions/sink-energy.md): the reference module's unit tests (a - # bracket summary is its intervals in integers, a round's figure and least declaration - # are energy-measurement §6.2's — checked against energy_report's reference — a round is - # refused by its first broken rule, a row seals its median and covers its spread, a - # segment's bounded window has no count term, the estimators and shares round half up, - # records sign and their identities cover the signature, §8 lists exactly the codes), the - # readers' unit tests (a counter is live only as a live rail advances, an advance crosses - # the wrap; on macOS, the per-process counter reads this process) and - # tests/sink_energy.rs (the committed corpus regenerates byte for byte from its fixed - # seeds, every case gets its stated outcome, every reason of §8 refuses a case, and every - # measured table under tables/ is accepted with its readings and procedure); then the - # corpus's second implementation, sink_energy_verify.py, written from the extension's text - # and sharing no code with wai-rs, which recomputes every round, row, segment and split in - # integers and requires every mutated copy of the corpus to fail; standard library only, - # run isolated (-I). These are the integer checks; the counters themselves are read by - # energy-hardware.yml's sink jobs, on the runners that have them. - - name: sink energy · wai-rs and sink_energy_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib sink_energy::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib sink_meter::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test sink_energy) - (cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_sink_energy_vectors" verify ../live-conformance/sink-energy) - python3 -I live-conformance/sink_energy_verify.py live-conformance/sink-energy - # wai.live.moderation (extensions/live-moderation.md): the reference module's unit tests - # (policies, statements and notices round-trip and their rules refuse, the export's - # strings and dates, the content types, the states that show an item) and - # tests/live_moderation.rs (every file of the committed corpus, vectors.json and each - # statement-of-reasons export, regenerates byte for byte from its fixed seeds and no - # other file is there, wai-rs gives every room, export and clip its stated outcome, and - # §10 lists exactly the reference's codes, each of which the corpus gives); then the - # corpus's second implementation, live_moderation_verify.py, written from the - # extension's text and sharing no code with wai-rs, which verifies each room with - # live_rooms_verify.py and each clip with live_record_verify.py, writes every export - # byte for byte, reads the extension's category, content-type and reason tables, and - # requires every mutated copy of the corpus to fail; standard library only, run - # isolated (-I). - - name: live moderation · wai-rs and live_moderation_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_moderation::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_moderation) - python3 -I live-conformance/live_moderation_verify.py live-conformance/moderation - # wai.audio.opus.dred.int (extensions/opus-dred.md): first the draft's weights file and - # test vectors, which the corpus names by SHA-256 and does not hold, fetched from the - # draft's locations and checked against its digests (live-conformance/dred/fetch.sh). - # Then the reference module's unit tests (the range decoder against its encoder, the - # extension draft's worked examples, payloads round-trip, weights quantized from their - # bits, the activation tables, the driver on a synthetic set), tests/dred_exact.rs (the - # corpus regenerates byte for byte from its seeds and the weights and no other file is - # there, every payload and case reaches its outcome, §8 lists exactly the module's codes, - # §4 states its digests), and in release tests/dred_exact_measure.rs (every record of the - # draft's test vectors decoded through the profile and through the floating-point - # network: the profile passes the draft's feature test, and §6's table is what it - # measures). Then the corpus's second implementation, dred_verify.py, written from the - # extension's text and sharing no code with wai-rs, which recomputes every outcome, - # digest and row in its own integer arithmetic and requires every mutated copy of the - # corpus to fail; standard library only, run isolated (-I). - - name: opus deep redundancy · wai-rs and dred_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - d="${RUNNER_TEMP:-/tmp}/dred" - bash live-conformance/dred/fetch.sh "$d" vectors - w="$d/dred_unquantized_weights.bin" - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib dred_exact::) - (cd wai-rs && WAI_DRED_WEIGHTS="$w" cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test dred_exact) - (cd wai-rs && WAI_DRED_WEIGHTS="$w" WAI_DRED_VECTORS="$d" cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test dred_exact_measure) - python3 -I live-conformance/dred_verify.py live-conformance/dred "$w" - # wai.live.ads (extensions/live-ads.md): the reference module's unit tests (policies and - # annotations round-trip and their rules refuse, a break's identity, tags, events and - # asset lists written exactly), the break form of wai.video.ssai's receipt (its JSON - # carriage, each of its rules, its strict signature, the stitched form's legacy pin - # unchanged), and tests/live_ads.rs (every file of the committed corpus, vectors.json and - # each room's HLS tags, regenerates byte for byte from its fixed seeds and no other file - # is there, wai-rs gives every room its stated outcome, and §10 lists exactly the - # reference's codes, each of which the corpus gives); then the corpus's second - # implementation, live_ads_verify.py, written from the extension's text and sharing no - # code with wai-rs, which verifies each room with live_rooms_verify.py, writes every tag, - # event and asset list byte for byte, verifies every receipt with WAI's strict Ed25519 - # rule and BLAKE3 from claim-conformance, reads the extension's reason table, and - # requires every mutated copy of the corpus to fail; standard library only, run isolated - # (-I). - - name: live ads · wai-rs and live_ads_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_ads::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib wai_video_ssai::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_ads) - python3 -I live-conformance/live_ads_verify.py live-conformance/ads - # The proxy form of wai.video.keys (extensions/key-release.md): the reference module's unit - # tests (the commitment list, the entitlement and the ticket round-trip and their rules - # refuse, each check of the decision in its order, offline licences and renewals, each own - # rule of the receipt, its strict signature, the CPIX reader and the derivation of a list - # from a CPIX document), form 1's unit tests (its legacy pin unchanged), and - # tests/key_release.rs (every file of the committed corpus regenerates byte for byte from - # its fixed seeds and no other file is there, every log gets its stated outcome, and the - # extension's tables are the reference's codes); then the corpus's second implementation, - # key_release_verify.py, written from the extension's text and sharing no code with wai-rs, - # which reads the extension's tables, derives every list from its CPIX document with the - # standard library's XML parser, recomputes every decision of every log, verifies every - # signature with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, and requires - # every mutated copy of the corpus to fail; standard library only, run isolated (-I). - - name: key release · wai-rs and key_release_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib key_release::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib wai_video_keys::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test key_release) - python3 -I live-conformance/key_release_verify.py live-conformance/keys - # wai.live.ad_presentation (extensions/ad-presentation.md): the reference module's unit - # tests (claims round-trip and their rules refuse, the quartile, the qualifying run, the - # touched objects, a policy's presentation fields, RFC 9577's TokenChallenge) and - # tests/ad_presentation.rs (every file of the committed corpus regenerates byte for byte - # from its fixed seeds and no other file is there, RFC 9578's own token verifies under its - # key, wai-rs gives every room its outcome and every claim its verdict, and §11 lists - # exactly the reference's codes); then the corpus's second implementation, - # ad_presentation_verify.py, written from the extension's text and sharing no code with - # wai-rs, which verifies each room with live_ads_verify.py, judges every claim and its - # RSASSA-PSS itself, requires every mutated copy of the corpus to fail and catches each of - # its own checks dropped; standard library only, run isolated (-I). Last, the browser - # client's reference producer (joule-moq-web/web/ads), which records observation - # sequences into spans and writes and signs the corpus's claims from them, byte for byte. - - name: ad presentation · wai-rs, ad_presentation_verify.py and the reference producer - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_ads::presentation::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test ad_presentation) - python3 -I live-conformance/ad_presentation_verify.py live-conformance/presentation - node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/ads/presentation.test.mjs - # wai.video.stitched_presentation (extensions/stitched-presentation.md): the reference - # module's unit tests (claims and pod-input records round-trip and their rules refuse, - # markers' bodies, a CPM price scaled to one impression, the XML subset) and the - # live-compose ad frames' (a squeeze's window and L tile the canvas, a double box's - # boxes), and tests/stitched_presentation.rs (every file of the committed corpus - # regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives - # every record, claim and ad its stated facts and every vector its spend by currency, and - # §9 lists exactly the codes the reference adds); then the tool's own check of the corpus - # through its binary; then the corpus's second implementation, - # stitched_presentation_verify.py, written from the extension's text and sharing no code - # with wai-rs, which reads every playlist, asset list, MPD, VAST document, form-1 receipt, - # record and claim itself, requires every mutated copy of the corpus to fail and catches - # each of its own checks dropped; standard library only, run isolated (-I). Last, the - # browser client's stitched producer (joule-moq-web/web/ads/stitched.js), which reads the - # markers from the corpus's manifests, measures an ad's region, and writes and signs the - # corpus's claims, byte for byte. - - name: stitched presentation · wai-rs, stitched_presentation_verify.py and the reference producer - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib stitched_presentation::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_compose::layout::frame_tests) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test stitched_presentation) - (cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_stitched_presentation" vectors verify ../live-conformance/stitched) - python3 -I live-conformance/stitched_presentation_verify.py live-conformance/stitched - node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/ads/stitched.test.mjs - # wai.video.lineup (extensions/channel-lineup.md): the reference module's unit tests (times, - # the document type declaration, the reasons) and tests/channel_lineup.rs (every file of the - # committed corpus regenerates byte for byte from its fixed seeds, and no other file is - # there; wai-rs gives every import and every verification its outcome; §10 lists exactly - # the reference's codes; §9's commands, run as the text writes them through `wai`, yield - # the corpus's receipt, lineup, cues and guides, and a refusal is its reason with exit 1); - # then the corpus's second implementation, channel_lineup_verify.py, written from the - # extension's text and sharing no code with wai-rs, which reimports every vector, requires - # every outcome and fact, and requires every mutated copy of the corpus to fail and every - # check it drops to be caught. - - name: channel lineup · wai-rs and channel_lineup_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib channel_lineup::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test channel_lineup) - (cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_channel_lineup_vectors" verify ../live-conformance/channels) - python3 -I live-conformance/channel_lineup_verify.py live-conformance/channels - # wai.live.provenance (extensions/live-provenance.md): the reference module's unit tests - # (boxes, chunks, emsg and the BMFF hash; the JUMBF store; COSE_Sign1 and the OKP key; - # certificates written and read, their profile and chain; identifiers, track renderings, - # dates and the compositor's source type) and tests/live_provenance.rs (every file of the - # committed corpus regenerates byte for byte from its fixed seeds and the packaging - # corpus's AV1 floor, and no other file is there; wai-rs gives every vector its stated - # outcome; every reason of §12 refuses a vector; §12 lists exactly the reference's codes - # with the two C2PA codes it reports); then the corpus's second implementation, - # c2pa_live_verify.py, written from the extension's text and sharing no code with - # wai-rs, which reads CBOR, ISO BMFF, JUMBF, COSE and X.509 DER itself, verifies every - # signature with WAI's strict Ed25519 rule and every receipt with BLAKE3 from - # claim-conformance, reads the extension's reason table, and requires every mutated copy - # of a good vector to be refused; standard library only, run isolated (-I). - - name: live provenance · wai-rs and c2pa_live_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_provenance::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_provenance) - python3 -I live-conformance/c2pa_live_verify.py live-conformance/c2pa-live - # wai.media.synthesis (extensions/synthesis.md): the reference module's unit tests (the - # pinned derivations by their rules, the class lattice, a generator's envelope) and - # tests/synthesis.rs (the committed corpus regenerates byte for byte from its fixed seeds - # and no other file is there, wai-rs gives every case its stated outcome and statements, - # §12 lists exactly the reference's codes, each of which the corpus gives, and §10.2's - # table pins exactly the reference's icons); then the corpus's second implementation, - # synthesis_verify.py, written from the extension's text and sharing no code with wai-rs, - # which runs the pinned derivations and writes each digital source type and C2PA - # statement itself, verifies each room with live_rooms_verify.py and every record with - # WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, reads the extension's icon - # and reason tables, and requires every mutated copy of the corpus to fail; standard - # library only, run isolated (-I). - - name: synthesis provenance · wai-rs and synthesis_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib synthesis::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test synthesis) - python3 -I live-conformance/synthesis_verify.py live-conformance/synthesis - # The registered watermark schemes wai-mark:tent64 and wai-mark:tent64s - # (extensions/synthesis.md §7.4 to §7.9): the reference's unit tests (the code corrects 29 - # errors and its generator is the definition's, the layouts, the embedding and the - # detector in every orientation, the tiled detector's exact autocorrelation and its - # recovery from a crop and a rotation, the battery's resampling, rotation and keyframe), - # then tests/synthesis_mark.rs in release, since it embeds and attacks every picture of - # the measurement sets under both schemes (every file of live-conformance/synthesis-mark/ - # regenerates byte for byte, both measurements among them, and no other file is there; - # every embedding, detection and case gets what it states; the extension's definitions, - # generator polynomial and shuffles are the reference's; §7.5.8 and §7.9.7 claim only - # attacks the measurements show survived, and §7.5.6's and §7.9.5's figures are the - # measurements'); then the second implementation, synthesis_mark_verify.py with - # _wai_mark.py, written from the extension's text and sharing no code with wai-rs, which - # embeds, detects and decodes itself, both stages of the tiled detector among them, runs - # the battery's picture attacks, derives the frames the vectors name, verifies every case - # with synthesis_verify.py, mutates copies of the corpus and weakens its own verifier in - # turn; standard library only, run isolated (-I). - - name: synthesis watermark · wai-rs and synthesis_mark_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- synthesis::mark synthesis_mark::) - (cd wai-rs && cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test synthesis_mark) - python3 -I live-conformance/synthesis_mark_verify.py live-conformance/synthesis-mark - # wai.media.upscale (extensions/exact-upscale.md): the integer super-resolution - # derivations' unit tests (the wai-ops/1 graph against the arithmetic of §3.2 written out - # loop by loop, on random priors and pictures; a prior with no detail is the pinned - # upscale; sr4 is sr2 twice; a prior that breaks §3.1 refused in its order; a picture past - # the caps refused), the module's unit tests (the pixel counts of §6.3, a residual within - # half its step), and tests/exact_upscale.rs (the committed corpus regenerates byte for - # byte from its fixed seeds and its two pinned trained priors, and no other file is there; - # wai-rs gives every case its stated outcome and statements; §8 lists exactly the - # reference's codes, each of which the corpus gives; sha256.json names every accepted - # output); then the corpus's second implementation, upscale_verify.py, written from the - # extension's text and sharing no code with wai-rs, which runs the derivations with - # _int_sr.py, verifies the claims with synthesis_verify.py and the residuals with - # staged_verify.py, and requires every mutated copy of the corpus to fail; standard - # library only, run isolated (-I). - - name: exact upscale · wai-rs and upscale_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib int_sr::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib exact_upscale::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test exact_upscale) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test cli_derive) - python3 -I live-conformance/upscale_verify.py live-conformance/upscale - # Robustness evidence for soft bindings (extensions/robustness-evidence.md): the - # reference's unit tests (gamma is the rounded power, rates are exact and reduced, every - # attack of synthesis §7.5.7's battery is a transform that gives its bytes, parameters - # outside their ranges are refused), then tests/robustness.rs in release, since - # regenerating the corpus runs the harness over the measurement sets of synthesis §7.5.8 - # (every file of live-conformance/robustness/ regenerates byte for byte and no other file - # is there; every case gets what it states; §12 lists exactly the reference's codes and - # the corpus refuses a case for each; §4 registers exactly the reference's transforms; - # the tent64 records' tables are the synthesis corpus's battery.json; the tent64 claim is - # synthesis §7.5.8's list; jwp-receipts §3.1 lists the span reasons); then the second - # implementation, robustness_verify.py with _wai_mark.py, written from the extension's - # text and sharing no code with wai-rs, which reruns the conformance record's battery end - # to end and requires the same table, checks the tent64 records against battery.json and - # the claim against synthesis §7.5.8, mutates copies of the corpus and weakens its own - # verifier in turn; standard library only, run isolated (-I). - - name: robustness evidence · wai-rs and robustness_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib robustness::) - (cd wai-rs && cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test robustness) - python3 -I live-conformance/robustness_verify.py live-conformance/robustness - # Tensor files as prior forms (extensions/prior-carriage.md §9 to §14): the module's unit - # tests (each layout read strictly and written canonically, the arithmetic coder's integers - # round-tripping and a damaged payload caught, the profile's checks in their order, both - # bindings through every form) and tests/prior_wire.rs (the committed corpus regenerates - # byte for byte from its pinned native artifacts; §13 lists exactly the reference's codes; - # every corpus prior in every form loads to its native bytes, and the two trained WSR1 - # priors give the same upscaled picture as their native files); then the corpus's second - # implementation, prior_forms_verify.py, written from the extension's text and sharing no - # code with wai-rs, which also requires every byte-flipped copy of an accepted file to be - # caught; standard library only, run isolated (-I). - - name: prior forms · wai-rs and prior_forms_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- prior_wire:: strict_json:: codecs::prior_form) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test prior_wire) - python3 -I live-conformance/prior_forms_verify.py live-conformance/prior-forms - # Scene carriage (extensions/scene-carriage.md): the module's unit tests (the conversions - # and spellings of §3 against fixed values, the containers an import builds equal to the - # ones wai-rs's world and audio-scene modules write, base64 and the USD subset's reader) - # and tests/scene_map.rs (the committed corpus regenerates byte for byte: every bundle - # written as glTF and USD and read back, every refusal reached; §8 lists exactly the - # reference's codes); then the corpus's second implementation, scene_verify.py, written - # from the extension's text and sharing no code with wai-rs, which also requires every - # byte-flipped copy of an exported document to be caught; standard library only, run - # isolated (-I). - - name: scene carriage · wai-rs and scene_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- scene_map:: strict_json::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test scene_map) - python3 -I live-conformance/scene_verify.py live-conformance/scene - # wai.avatar.face (extensions/avatars.md §6 to §13): the module's unit tests (the - # generator's wai-ops/1 graph against §7.6's arithmetic written out loop by loop; the warp - # at rest is the reference; the mask's table; the files read and refused in their order), - # and tests/avatar_face.rs (the committed corpus regenerates byte for byte from its fixed - # seeds and its pinned inputs, and no other file is there; wai-rs gives every form, frame - # and case what it states; §10 lists exactly the reference's codes; §7.9's figures are - # training.json's; the registered generator meets acc64 and not acc32); then the corpus's - # second implementation, avatar_face_verify.py, written from the extension's text and - # sharing no code with wai-rs, which runs the driver and the generator's arithmetic in - # Python's integers, verifies the claims, the log and the label policy with - # synthesis_verify.py, and requires every mutated copy of the corpus to fail; standard - # library only, run isolated (-I). - - name: faces from keypoints · wai-rs and avatar_face_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib avatar_face::) - (cd wai-rs && cargo test --release --no-default-features --features "$WAI_RELEASE_FEATURES" --test avatar_face) - python3 -I live-conformance/avatar_face_verify.py live-conformance/avatar-face - # wai.ai.usage (extensions/ai-usage.md): the reference module's unit tests (RFC 9651's - # Dictionary parser against the vocabulary's examples and the RFC's edges, statements - # read and refused in their steps' order, the precedence rule, HTTP fields compared by - # meaning), the consumption class's body and terms in claim_receipt, and - # tests/ai_usage.rs (vectors.json regenerates byte for byte from its fixed seeds and no - # other file is there, wai-rs gives every statement, object, room, response, catalog, - # claim and gate its stated outcome, §9 lists exactly the reference's codes, each of which - # the corpus gives, and jwp-receipts §3.1 lists the class rules); then the corpus's second - # implementation, ai_usage_verify.py, written from the extension's text and sharing no - # code with wai-rs, which parses Structured Fields, canonical forms, receipts and claims - # itself, reads rooms with live_rooms_verify.py, checks signatures with WAI's strict - # Ed25519 rule and BLAKE3 from claim-conformance, reads the extension's reason table, - # requires every mutated copy of the corpus to fail, and drops each of its own checks in - # turn and requires the corpus to catch the weakened verifier; standard library only, run - # isolated (-I). Last, claim-conformance's own verifier reads every consumption claim of - # the corpus through jwp-receipts §3.1 steps 1 to 4 and requires the stated step and reason. - - name: AI usage · wai-rs and ai_usage_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib ai_usage::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib claim_receipt::consumption::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test ai_usage) - python3 -I live-conformance/ai_usage_verify.py live-conformance/usage - python3 -I claim-conformance/claim_verify.py consumption live-conformance/usage/vectors.json - # wai.media.contribution (extensions/contribution.md): the reference module's unit tests - # (every proof of the steps' tree walks to its root, the copying operations follow their - # sample mappings, a map has one encoding) and tests/contribution.rs (the committed corpus - # regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives - # every case its stated outcome and statements, §11 lists exactly the reference's codes, - # each of which the corpus gives, and the corpus generates with the model file of - # wai.generate.spectral's conformance vector); then the corpus's second implementation, - # contribution_verify.py, written from the extension's text and sharing no code with - # wai-rs, which evaluates every pinned operation, runs the generator in its own arithmetic - # after checking it against the generator's conformance vector, computes every map, share - # and C2PA statement itself, reads origin claims with synthesis_verify.py and every record - # with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, requires every mutated - # copy of the corpus to fail, and drops each of its own checks in turn and requires the - # corpus to catch the weakened verifier; standard library only, run isolated (-I). - - name: contribution records · wai-rs and contribution_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib contribution::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test contribution) - python3 -I live-conformance/contribution_verify.py live-conformance/contribution - # wai.live.contribution (extensions/live-contribution.md): the link records' corpus - # under live-conformance/links/, which joule-moq-ingest's reference generates and its - # own tests check (joule-code-conformance.yml); here its second implementation, - # live_contribution_verify.py, written from the extension's text and sharing no code - # with the reference, reads every part and series itself, checks every signature with - # WAI's strict Ed25519 rule from claim-conformance, requires each vector's stated outcome - # and reason and every reason of §7's last list, requires every mutated copy to be - # refused, and drops each of its own checks in turn and requires the corpus to catch the - # weakened verifier; standard library only, run isolated (-I). - - name: live contribution · live_contribution_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - python3 -I live-conformance/live_contribution_verify.py live-conformance/links - # wai.delivery.settlement (extensions/settlement.md): the reference module's unit tests - # (range proofs over every range around every chunk boundary of objects of many lengths - # against BLAKE3's own root, tampered proofs refused, the group tree of jwp-receipts §2.2, - # records and item lists read and refused, amounts rounded down per line) and - # tests/settlement.rs (vectors.json regenerates byte for byte from its fixed seeds and no - # other file is there, wai-rs gives every statement and range proof its stated outcome, §11 - # lists exactly the reference's codes, each of which the corpus gives, beside every code it - # reuses); then the corpus's second implementation, settlement_verify.py, written from the - # extension's text and sharing no code with wai-rs, which verifies range proofs over - # BLAKE3's chunk tree with claim-conformance's BLAKE3, reads rooms, grants and proofs with - # live_rooms_verify.py and per-hop records with live_metering_verify.py, checks signatures - # with WAI's strict Ed25519 rule, recomputes every figure of every statement, reads the - # extension's reason table, requires every mutated copy of the corpus to fail, and drops - # each of its own checks in turn and requires the corpus to catch the weakened verifier; - # standard library only, run isolated (-I). - - name: delivery settlement · wai-rs and settlement_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib settlement::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test settlement) - python3 -I live-conformance/settlement_verify.py live-conformance/settlement - # wai.delivery.ledger (extensions/receipt-ledger.md): the reference module's unit tests and - # tests/receipt_ledger.rs (vectors.json regenerates byte for byte from its fixed seeds and - # from the sources it takes from the metering, settlement, ads, presentation, stitched and - # routes corpora and the packs of pack-conformance, and no other file is there; wai-rs gives every vector its - # stated outcome; every reason of §10 and every reused code refuses a vector, and so does - # each source kind's own extension; §10 lists exactly the reference's codes; under a tariff - # of version 3, jobs priced per operation and a chain that bills a job's work once, its - # output's receipt cited again by a re-run, with version 2 billing as before); then the - # corpus's second implementation, receipt_ledger_verify.py, written from the extension's - # text and sharing no code with wai-rs, which verifies each source with that corpus's own - # second implementation (live_metering_verify.py, settlement_verify.py, live_ads_verify.py - # with ad_presentation_verify.py, stitched_presentation_verify.py, pack_verify.py, and for a - # media job route_record_verify.py and _int_motion_receipt.py) and - # reports a stitched stream's statement as that verifier gives it, checks signatures with - # WAI's strict Ed25519 rule, recomputes every line, citation, energy row and report, - # requires every mutated copy of the corpus to fail, and drops each of its own checks in turn and requires - # the corpus to catch the weakened verifier; run isolated (-I), with the `cryptography` package - # installed above for the OER/2 reference verifier that checks a job's hop receipts. - - name: receipt ledger · wai-rs and receipt_ledger_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib receipt_ledger::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test receipt_ledger) - python3 -I live-conformance/receipt_ledger_verify.py live-conformance/ledger - # wai.delivery.revenue_share (extensions/revenue-share.md): the reference module's unit tests - # (the largest-remainder division and its ties, the date-time a marker states, the contract's - # and the statement's reading and rules, contracts that overlap) and the channel receipt's - # JSON carriage; tests/revenue_share.rs (vectors.json regenerates byte for byte from its fixed - # seeds and the ledger corpus, and no other file is there; wai-rs gives every vector its - # stated outcome; every reason of §11 and every reused code refuses a vector; §11 lists exactly - # the reference's codes; the worked example, three statements of one month computed and then - # recomputed from the public records alone; the four operations from the command line); then - # the corpus's second implementation, revenue_share_verify.py, written from the extension's - # text and sharing no code with wai-rs, which verifies each ledger document with - # receipt_ledger_verify.py, reads every contract, run, marker and statement itself, - # recomputes every attribution, line, division and citation, requires every mutated copy of - # the corpus to fail, and drops each of its own checks in turn and requires the corpus to - # catch the weakened verifier; standard library only, run isolated (-I). - - name: revenue share · wai-rs and revenue_share_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib revenue_share::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib wai_video_channel::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test revenue_share) - python3 -I live-conformance/revenue_share_verify.py live-conformance/share - # wai.delivery.opportunity (extensions/opportunity-report.md): the reference module's unit - # tests (the SCTE 35 subset reader against the standard's sample and damaged sections, the - # reported statement's JSON and CSV forms and dates, the report's and the check's reading and - # rules, the ratios); tests/opportunity_report.rs (vectors.json regenerates byte for byte - # from its fixed seeds and no other file is there; wai-rs gives every vector its stated - # outcome; every reason of §13 and every reused code refuses a vector; §13 lists exactly the - # reference's codes; the worked example, a channel's day whose reported statement - # under-reports one owner's programme, and the check that shows it, recomputed from the - # public records alone); then the corpus's second implementation, opportunity_report_verify.py, - # written from the extension's text and sharing no code with wai-rs, which verifies rooms - # with live_ads_verify.py and contracts, runs and signed statements with - # revenue_share_verify.py, reads every report, check, section and reported statement itself, - # recomputes every break, attribution, line, citation and flag, requires every mutated copy - # of the corpus to fail, and drops each of its own checks in turn and requires the corpus to - # catch the weakened verifier; standard library only, run isolated (-I). - - name: opportunity report · wai-rs and opportunity_report_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib opportunity_report::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test opportunity_report) - python3 -I live-conformance/opportunity_report_verify.py live-conformance/opportunity - # wai.delivery.audit (extensions/stream-audit.md): the reference module's unit tests (the - # acknowledgement windows, chains and repeats of §4.2, the origin rule, the report's bytes, - # channel names) and tests/stream_audit.rs (vectors.json regenerates byte for byte from its - # fixed seeds and no other file is there, wai-rs gives every session and report its stated - # outcome, §14 lists exactly the reference's codes, each of which the corpus gives, beside - # every code it borrows); then the corpus's second implementation, stream_audit_verify.py, - # written from the extension's text and sharing no code with wai-rs, which reads - # acknowledgements with settlement_verify.py, claims with ad_presentation_verify.py and logs - # and energy classes with live_metering_verify.py, checks every signature with WAI's strict - # Ed25519 rule, recomputes every report, requires every mutated copy of the corpus to fail, - # and drops each of its own checks in turn and requires the corpus to catch the weakened - # verifier; standard library only, run isolated (-I). The collector and the player - # integration run in wai-audit.yml. - - name: stream audit · wai-rs and stream_audit_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib stream_audit::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test stream_audit) - python3 -I live-conformance/stream_audit_verify.py live-conformance/audit - # wai.delivery.fleet (extensions/fleet.md): the reference module's unit tests (the enrolment's - # rules, a summary line split only in the relay's form, a row summed across classes told from - # a wrong one, the four coverages) and tests/fleet.rs (vectors.json regenerates byte for byte - # from its seeds and the metering corpus and no other file is there; wai-rs gives every - # vector its stated outcome; every reason of §10 and every reused code refuses a vector; the - # accepted vectors state measured, unmetered and partial energy as §5.4 does; the week's - # records held as a service holds them sign the corpus's report byte for byte; §10 lists - # exactly the reference's codes); then the corpus's second implementation, fleet_verify.py, - # written from the extension's text and sharing no code with wai-rs, which reads per-hop - # records with live_metering_verify.py, checks every signature with WAI's strict Ed25519 - # rule, recomputes every line, row, unmetered count and citation, requires every mutated - # copy of the corpus to fail, and drops each of its own checks in turn and requires the - # corpus to catch the weakened verifier; standard library only, run isolated (-I). The - # service and its smoke checks over HTTP run in wai-fleet.yml. - - name: fleet · wai-rs and fleet_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib fleet::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test fleet) - python3 -I live-conformance/fleet_verify.py live-conformance/fleet - # wai.route.record (extensions/route-record.md): the reference module's unit tests (a - # document read strictly and held to the subset, §2's encodings, an estimate past the subset - # unknown, a null ranked last, the expected figures per successful job and when they are - # unknown, the outputs' commitment, §8 listing exactly the reference's codes) and - # tests/route_record.rs (vectors.json regenerates byte for byte from its seeds and no other - # file is there; wai-rs gives every vector its stated outcome; every reason of §8 and every - # reused code refuses a vector; the accepted vectors are §10's, of version 1 and of version 2, - # every limit removing a candidate; a record is bound to each of its documents; `wai route - # verify` prints the outcome and exits by it); then the corpus's second implementation, - # route_record_verify.py, written from the extension's text and sharing no code with wai-rs, - # which checks the router's signature with WAI's strict Ed25519 rule and the hop receipts - # with the OER/2 reference verifier (the `cryptography` package, installed above), computes - # the outputs' commitment with claim-conformance/_blake3.py, recomputes every candidate, - # removal, expected figure, rank, order, decision and attempt, requires every mutated copy - # of the corpus to fail, and drops each of its own checks in turn and requires the corpus to - # catch the weakened verifier; run isolated (-I). - - name: route record · wai-rs and route_record_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib route_record::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test route_record) - python3 -I live-conformance/route_record_verify.py live-conformance/routes - # The native sink (extensions/native-sink.md): the reference module's unit tests (dates, - # durations and attribute lists read as a player reads them, origins serialized as RFC 6454 - # does, names that state a content hash and names that do not, the acknowledgements - # rebuilt from hashes alone equal to stream-audit's reference, a bundle that verifies and - # one refused for another owner or channel) and tests/native_sink.rs (vectors.json - # regenerates byte for byte from its seeds and no other file is there, every call gets the - # answer it states, §8 lists exactly the reasons the corpus reaches, and every case run - # through the C interface's symbols gets the same answers); then the corpus's second - # implementation, native_sink_verify.py, written from the extension's text and sharing no - # code with wai-rs, which replays every case, reads playlist receipts and their chain with - # http_verify.py, holds every bundle to its own session field by field and verifies it with - # stream_audit_verify.py, requires every mutated copy of the corpus to fail, and drops each - # of its own checks in turn and requires the corpus to catch the weakened verifier; - # standard library only, run isolated (-I). The Swift package runs in wai-sink-swift.yml. - - name: native sink · wai-rs and native_sink_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib native_sink::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test native_sink) - python3 -I live-conformance/native_sink_verify.py live-conformance/native-sink - # The browser sink (extensions/browser-sink.md): the reference module's unit tests (the - # registry gate's rows, registered tiers, integers as §5 reads them, a binding's - # canonical form, an answer for an unregistered codec string never read, powerEfficient - # never stated off the user agent's route, the side table's ambiguity) and - # tests/browser_sink.rs (both files of the committed corpus regenerate byte for byte and - # no other file is there, wai-rs gives every case its stated outcome, every reason of - # §11 appears in the corpus and §11 lists exactly the reference's codes, and the chain - # the generator signs by hand verifies under jwp_delivery); wai-live-web's browser - # functions held to the same corpus natively, and its wasm-route decoder verifying, - # decoding and binding every chain object to the published frame digests; then the - # corpus's second implementation, browser_sink_verify.py, written from the extension's - # text and sharing no code with wai-rs, which reads the registry export for each - # capability's class, recomputes every route, evaluation, energy entry, binding, chain - # signature, frame state and side table, changes the answers for unregistered codec - # strings every way and requires the corpus still to pass, and requires every mutated - # copy of the corpus to fail; standard library only, run isolated (-I). The browser's - # own run of it is joule-moq-web.yml's scenario binding. - - name: browser sink · wai-rs, wai-live-web and browser_sink_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib browser_sink::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test browser_sink) - (cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_browser_sink_vectors" verify ../live-conformance/browser) - (cd wai-live-web && cargo test --locked --features int_video --lib browser_tests) - python3 -I live-conformance/browser_sink_verify.py live-conformance/browser - # wai.media.takedown (extensions/takedown.md): the reference module's unit tests (times, - # clock identifiers, the published registry, the order of subjects, the lower median) and - # tests/takedown.rs (the committed corpus regenerates byte for byte from its fixed seeds - # and no other file is there, wai-rs gives every case its stated outcome and facts and - # every counts entry its rows, §11 lists exactly the reference's codes, each of which the - # corpus gives, and the registry the module carries is registry/takedown-clocks.json); - # then the corpus's second implementation, takedown_verify.py, written from the - # extension's text and sharing no code with wai-rs, which computes every closure, holder, - # purge time, clock verdict and counts row itself, reads derivation records and delivery - # receipts with edge_verify.py, origin claims with synthesis_verify.py, acknowledgements - # with settlement_verify.py and every record with WAI's strict Ed25519 rule and BLAKE3 - # from claim-conformance, requires every mutated copy of the corpus to fail, and drops - # each of its own checks in turn and requires the corpus to catch the weakened verifier; - # run isolated (-I). The reference relay's DVR store honours takedowns (§10.1), tested by - # joule-moq-relay's tests/dvr.rs in joule-code-conformance.yml. - - name: takedown · wai-rs and takedown_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib takedown::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test takedown) - python3 -I live-conformance/takedown_verify.py live-conformance/takedown - # wai.media.influence (extensions/influence.md): the reference module's unit tests (the - # blend keeps its base and its references by weight, the makers' split is whole, an - # estimate mixed into a share is told from another difference) and tests/influence.rs (the - # committed corpus regenerates byte for byte from its fixed seeds and no other file is - # there, wai-rs gives every case, statement and report its stated outcome, every reason of - # §9 and every code it reuses refuses a vector, and §9 lists exactly the reference's codes); - # then the corpus's second implementation, influence_verify.py, written from the - # extension's text and sharing no code with wai-rs, which verifies each session with - # contribution_verify.py, regenerates every conditioned step with that implementation's - # arithmetic of the generator, reads usage statements with ai_usage_verify.py and - # statements with settlement_verify.py, computes every step map, share, attribution line - # and report itself, requires every mutated copy of the corpus to fail, and drops each of - # its own checks in turn and requires the corpus to catch the weakened verifier; standard - # library only, run isolated (-I). - - name: influence attribution · wai-rs and influence_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib influence::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test influence) - python3 -I live-conformance/influence_verify.py live-conformance/influence - # wai.media.direction (extensions/direction.md): the reference module's unit tests (the - # interpolations reach both keys and stay between them, the framing measure at its bounds, - # a signed integer's one spelling, the timeline's starts and overlaps, a frame set's one - # encoding) and tests/direction.rs (the committed corpus regenerates byte for byte from its - # fixed seeds and no other file is there, wai-rs gives every case its stated outcome and - # statements, §13 lists exactly the reference's codes, each of which the corpus gives, and - # the corpus generates with the model file of wai.generate.spectral's conformance vector); - # then the corpus's second implementation, direction_verify.py, written from the - # extension's text and sharing no code with wai-rs, which runs the generator in its own - # arithmetic after checking it against the generator's conformance vector, renders every - # frame, interpolates every track, measures every box and writes every statement and C2PA - # statement itself, reads records with synthesis_verify.py's strict JSON reader and every - # signature with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, requires every - # mutated copy of the corpus to fail, and drops each of its own checks in turn and requires - # the corpus to catch the weakened verifier; standard library only, run isolated (-I). - - name: direction records · wai-rs and direction_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib direction::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test direction) - python3 -I live-conformance/direction_verify.py live-conformance/direction - # wai.live.coperformance (extensions/co-performance.md): the reference module's unit tests - # (the policy and every statement round-trip and their rules refuse, the attributes' - # decimal integers, a schedule's turns in bars and microseconds, the symbolic bodies read - # whole) and tests/co_performance.rs (vectors.json regenerates byte for byte from its fixed - # seeds and no other file is there, wai-rs gives every room its stated outcome, and §10 - # lists exactly the reference's codes, each of which the corpus gives); then the corpus's - # second implementation, co_performance_verify.py, written from the extension's text and - # sharing no code with wai-rs, which verifies each room with live_rooms_verify.py and every - # signature with WAI's strict Ed25519 rule from claim-conformance, recomputes every intent, - # fidelity statement, turn, floor, commitment, selection and credit, reads the extension's - # reason table, requires every mutated copy of the corpus to fail, and drops each of its - # own checks in turn and requires the corpus to catch the weakened verifier; standard - # library only, run isolated (-I). - - name: co-performance · wai-rs and co_performance_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib co_performance::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test co_performance) - python3 -I live-conformance/co_performance_verify.py live-conformance/coperformance - # wai.age.assurance (extensions/age-assurance.md): the reference module's unit tests (the - # pinned AlgorithmIdentifier and RFC 9578's first vector through this module, the - # TokenChallenge read exactly, the suppression floor, the grant attribute `age`, a span's - # coverage) and tests/age_assurance.rs (vectors.json regenerates byte for byte from its - # fixed seeds and no other file is there, wai-rs gives every case its stated outcome, and - # §11 lists exactly the reference's codes, each of which the corpus gives); then the - # corpus's second implementation, age_assurance_verify.py, written from the extension's - # text and sharing no code with wai-rs, which reads every record, writes JCS and verifies - # every token's RSASSA-PSS itself, reads rooms with live_rooms_verify.py, checks signatures - # with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, reads the extension's - # reason table, requires every mutated copy of the corpus to fail, and drops each of its - # own checks in turn and requires the corpus to catch the weakened verifier; standard - # library only, run isolated (-I). - - name: age assurance · wai-rs and age_assurance_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib age_assurance::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test age_assurance) - python3 -I live-conformance/age_assurance_verify.py live-conformance/age - # wai.bearer.receipts (extensions/bearer-receipts.md): the reference module's unit tests - # (the bound, the bitmap rules, the bearer classes, and the corpus reaching every stated - # outcome and failing a changed one) and tests/bearer_receipts.rs (vectors.json - # regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives - # every case its stated outcome, the cases that carry one receipt on every bearer class - # carry the same bytes with the same object verdicts, §10 lists exactly the reference's - # codes, each of which the corpus gives, and jwp-receipts §3.1 lists the class codes at - # their steps); then the corpus's second implementation, bearer_verify.py, written from - # the extension's text and sharing no code with wai-rs, which verifies groups and objects - # with jwp_verify.py, claims through jwp-receipts §3.1 steps 1-5 with claim_verify.py and - # confirmations with signer_verify.py (each needs the `cryptography` package installed - # above), writes the context checks, the verdict, the bound and offline confirmation - # itself, reads the extension's reason table, requires every mutated copy of the corpus - # to fail, and drops each of its own checks in turn and requires the corpus to catch the - # weakened verifier; run isolated (-I). Last, claim-conformance's own verifier reads every - # transmission and reconciliation claim of the corpus through jwp-receipts §3.1 steps 1 - # to 4 and requires the stated step and reason. - - name: bearer receipts · wai-rs and bearer_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib bearer) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test bearer_receipts) - python3 -I live-conformance/bearer_verify.py live-conformance/bearers - python3 -I claim-conformance/claim_verify.py bearer live-conformance/bearers/vectors.json - # wai.bearer.carousel (extensions/carousel.md): the reference module's unit tests (each code's - # bounds and division into source blocks, K', the passes of advance and repeat, plans, and the - # corpus reaching every stated outcome and failing a changed one) and tests/carousel.rs - # (vectors.json regenerates byte for byte from its fixed seeds, wai-rs gives every case its - # stated outcome, §9 lists exactly the reference's codes, Appendix A is its table of K', - # jwp-receipts §3.1 lists the class codes at their steps, K' and Partition[] agree with an - # independent RFC 6330 implementation for every block size, and each RaptorQ object a case - # holds available decodes with that implementation from what a receiver within the case's - # loss bound holds); then the corpus's second implementation, carousel_verify.py, written - # from the extension's text and sharing no code with wai-rs, which verifies groups and - # objects with jwp_verify.py and claims through steps 1-5 with claim_verify.py, writes the - # context checks, the availability and the plans itself, reads §9 and Appendix A, requires - # every mutated copy of the corpus to fail, and drops each of its own checks in turn; run - # isolated (-I). Last, claim-conformance's own verifier reads every claim of the corpus - # through jwp-receipts §3.1 steps 1 to 4 and requires the stated outcome. - - name: carousel schedule · wai-rs and carousel_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- carousel:: claim_receipt::schedule::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test carousel) - python3 -I live-conformance/carousel_verify.py live-conformance/carousel - python3 -I claim-conformance/claim_verify.py carousel live-conformance/carousel/vectors.json - # wai.delivery.holdings (extensions/holdings.md): the reference module's unit tests (an entry's - # one form, a record read back and every rule refusing, queries and a plan, and the corpus - # reaching every stated outcome and failing a changed one) and tests/holdings.rs (vectors.json - # regenerates byte for byte from its fixed seeds, wai-rs gives every case its stated outcome, - # and §7 lists exactly the reference's codes); then the corpus's second implementation, - # holdings_verify.py, written from the extension's text and sharing no code with wai-rs, which - # verifies group receipts with jwp_verify.py and signatures with the shared _ed25519.py, writes - # the record, its verdict, queries and plans itself, reads §7, requires every mutated copy of - # the corpus to fail and drops each of its own checks in turn; run isolated (-I). - - name: holdings · wai-rs and holdings_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- holdings::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test holdings) - python3 -I live-conformance/holdings_verify.py live-conformance/holdings - # wai.media.exploration (extensions/exploration.md): the reference module's unit tests (the - # descriptors of flat, single-row and checkerboard pictures, grid bins at both ends, the - # sample distance's zero, the taste score, derived prompts, and a schedule that replays and - # keeps the first comer without an objective) and tests/exploration.rs (vectors.json - # regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives - # every generation, descriptor vector and case its stated outcome, §13 lists exactly the - # reference's codes, each of which the corpus gives, and §3.2 names the reference's - # descriptors); then the corpus's second implementation, exploration_verify.py, written from - # the extension's text and sharing no code with wai-rs, which makes every picture with the - # generator as the extension's Appendix A restates it, computes the descriptors, replays - # every archive's schedule, reads usage statements with ai_usage_verify.py, verifies every - # record with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, reads the - # extension's reason table, requires every mutated copy of the corpus to fail, and drops each - # of its own checks in turn and requires the corpus to catch the weakened verifier; standard - # library only, run isolated (-I). - - name: exploration · wai-rs and exploration_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib exploration::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test exploration) - python3 -I live-conformance/exploration_verify.py live-conformance/exploration - # wai.video.encode (extensions/platform-encode.md): the reference module's unit tests (the - # Exp-Golomb reader, emulation prevention, NAL splitting, length-prefixed samples, the - # resampler, the source reader, the strict JSON scan and the date-time) and - # tests/platform_encode.rs (the fixture streams are the pinned bytes, every generated file - # regenerates byte for byte, every case gets its outcome, every record reason has a case, and - # §12 lists exactly the reference's reasons); then the corpus's second implementation, - # encode_verify.py, written from the extension's text and sharing no code with wai-rs or - # wai-pack, which reads every Annex B stream, its access units and parameter sets itself, - # verifies every record with WAI's strict Ed25519 rule and every object with BLAKE3 from - # claim-conformance, checks the encoded renditions against the base packaging golden, - # requires every mutated copy of the corpus to fail, and drops each of its own checks in turn - # and requires the corpus or its mutations to catch the weakened verifier; standard library - # only, run isolated (-I). The encoders themselves run on the macOS leg (codec-wrappers). - - name: platform-encode · wai-rs and encode_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --locked --no-default-features --features "$WAI_FEATURES" --lib platform_encode::) - (cd wai-rs && cargo test --locked --no-default-features --features "$WAI_FEATURES" --test platform_encode) - python3 -I encode-conformance/encode_verify.py encode-conformance - # wai.live.cowatch (extensions/co-watch.md): the reference module's unit tests (the policy - # and its rules, statements read back and refused, the controlled position in the order of - # §5.4, each branch of the catch-up rule, percentiles by nearest rank, the control attribute) - # and tests/co_watch.rs (vectors.json regenerates byte for byte from its fixed seeds and no - # other file is there, wai-rs gives every room and catch-up case its stated outcome, and §12 - # lists exactly the reference's codes, each of which the corpus gives); then the corpus's - # second implementation, co_watch_verify.py, written from the extension's text and sharing - # no code with wai-rs, which verifies each room with live_rooms_verify.py and every - # signature with WAI's strict Ed25519 rule, recomputes the controlled position, every - # sample, percentile and verdict, requires every mutated copy of the corpus to fail and - # drops each of its own checks in turn and requires the corpus to catch the weakened - # verifier; standard library only, run isolated (-I). Last, the browser client's co-watch - # module (joule-moq-web/web/room/cowatch.js), which gives every catch-up case its - # correction, recomputes each accepted room's controlled position, and writes the corpus's - # controls and presentation statements byte for byte. - - name: co-watch · wai-rs, co_watch_verify.py and the page's module - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib co_watch::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test co_watch) - python3 -I live-conformance/co_watch_verify.py live-conformance/cowatch - node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/room/cowatch.test.mjs - # wai.live.remote (extensions/remote-desktop.md): the reference module's unit tests (every - # statement read back and refused when a byte is short or over, the screen statement's - # field rules, the host's rules of §5.3 in their order, the grant attributes, a latency's - # legs, sealed actions: a seal with no plaintext in it that opens under the host's key - # alone, the commitment and what it binds, a seal that does not fit its actions, rules 8 - # and 9 after the others) and the crate's HPKE against RFC 9180 A.1.1; tests/remote_desktop.rs - # (vectors.json regenerates byte for byte from its fixed seeds and no other file is there, - # wai-rs gives every room its stated outcome at the commitment level and at the opened - # level, and §12 lists exactly the reference's codes, host refusals, floor refusals and - # the reasons a seal is not opened, each of which the corpus gives); - # then the corpus's second implementation, remote_desktop_verify.py, written from the - # extension's text and sharing no code with wai-rs, which verifies each room with - # live_rooms_verify.py, every signature with WAI's strict Ed25519 rule and every seal with - # the aggregation verifier's HPKE, recomputes every outcome from the rules, the floor and - # every latency's legs at both levels, requires every mutated copy of the corpus to fail and - # drops each of its own checks in turn and requires the corpus to catch the weakened - # verifier; standard library only, run isolated (-I). Last, the browser viewer's module - # (joule-moq-web/web/room/remote.js), which writes the corpus's input statements, floor - # statements and latency records byte for byte, seals every sealed input that opens again - # byte for byte, and gives every input the outcome the corpus states. - - name: remote desktop · wai-rs, remote_desktop_verify.py and the page's module - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- remote_desktop:: hpke::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test remote_desktop) - python3 -I live-conformance/remote_desktop_verify.py live-conformance/remote - node --test ../joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.test.mjs - # wai.telemetry.aggregate (extensions/telemetry-aggregation.md): the reference module's unit - # tests (the fields against a wide-arithmetic-free product, the generators' orders, - # TurboSHAKE128 against RFC 9861, HPKE against RFC 9180 A.1.1, the FLP accepting a valid - # measurement and refusing an invalid one, delivery-telemetry's key classes) and - # tests/telemetry_aggregate.rs (vectors.json regenerates byte for byte from its fixed seeds and - # no other file is there, vdaf-22/ is the draft's test vectors as published and wai-rs passes - # every one, every receipt, cell and delivery-only report gets its stated outcome, every - # reason of §11 and every reused code refuses a vector, §11 lists exactly the reference's - # codes, delivery-telemetry §3.4 classes exactly the reference's keys); then the corpus's - # second implementation, telemetry_aggregate_verify.py, written from the extension's text and - # the drafts it cites and sharing no code with wai-rs, which evaluates Prio3 itself and passes - # the draft's vectors, opens every report with its own X25519, HKDF, AES-128-GCM and HPKE, - # re-runs every flow (both aggregators' verification, the aggregation job byte for byte, the - # batch buckets, every refusal, the collection), verifies every receipt and cell with WAI's - # strict Ed25519 rule from claim-conformance, reads the key classes and the reason table from - # the texts, requires every mutated copy of the corpus to fail, and drops each of its own - # checks in turn and requires the corpus to catch the weakened verifier; standard library - # only, run isolated (-I). - - name: QoE aggregation · wai-rs and telemetry_aggregate_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- telemetry_aggregate:: telemetry::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test telemetry_aggregate) - python3 -I live-conformance/telemetry_aggregate_verify.py live-conformance/aggregate - # wai.media.tonemap, wai.meta.gainmap and wai.meta.st2094_50 (extensions/exact-tonemap.md): - # the reference module's unit tests (the fixed-point log2 and exp2 at known points, the - # table of K against its rule, the sRGB table, tan through its series, PCHIP slopes of a - # line, rounding of halves for both signs) and the PNG reader's, then tests/exact_tonemap.rs - # (the committed corpus regenerates byte for byte, no other file is there, every case and - # envelope gets its stated outcome, §9 lists exactly the reference's codes, §6 states the - # corpus's measurement against binary64, and frames.rgb8 is the int_video clip's decode - # whose digests its sidecar publishes); then the corpus's second implementation, - # tonemap_verify.py, written from the extension's text and sharing no code with wai-rs, - # which computes both derivations in integer arithmetic of its own, reads PNG, envelopes - # (with staged_verify.py) and both companions itself, recomputes §6's measurement, verifies - # every record with WAI's strict Ed25519 rule and BLAKE3 from claim-conformance, requires - # every mutated copy of the corpus to fail, and drops each of its own checks in turn and - # requires the corpus to catch the weakened verifier; standard library only, run isolated - # (-I). - - name: exact tone mapping · wai-rs and tonemap_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib -- exact_tonemap:: png_still::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test exact_tonemap) - python3 -I live-conformance/tonemap_verify.py live-conformance/tonemap - # wai.splat.exact (extensions/splat-exact.md): the reference module's unit tests - # (exp_neg_q16 against the exponential, binary32 ranges rounded half up exactly, the - # signed integers' grammar, the identity set carrying every byte, and tiles that leave a - # frame unchanged) and tests/splat_exact.rs (live-conformance/splat/ regenerates byte for - # byte and no other file is there, every case gets its stated outcome, §10 lists exactly the - # reference's codes, the tiles leave every view of every scene unchanged, and the draft - # schema names §3.2's members); then the corpus's second implementation, - # splat_exact_verify.py, written from the extension's text and sharing no code with wai-rs: - # it reads each glTF asset, decodes each payload with int_payload_verify.py's parser and - # rANS decoder and tools/wai_ops_check.py's whm1/1 interpreter, renders every view with its - # own integer renderer, verifies every record with WAI's strict Ed25519 rule and BLAKE3 - # from claim-conformance, reads the extension's reason table, requires every mutated copy - # of the corpus to fail, drops each of its own checks in turn and changes each rule of its - # renderer in turn, and requires the corpus to catch every one; standard library only, run - # isolated (-I). - - name: exact splats · wai-rs and splat_exact_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib splat_exact::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test splat_exact) - python3 -I live-conformance/splat_exact_verify.py live-conformance/splat - # wai.live.termination (extensions/stream-termination.md) and the display match of - # capability-constraints §4: the reference module's unit tests (sides without orientation, - # each exceedance alone, the least rendition alone where none is within, a writer that - # refuses after its termination) and tests/stream_termination.rs (the committed corpus - # regenerates byte for byte from its fixed seeds, wai-rs gives every match and log its - # stated outcome and facts, every reason refuses a log, §7 lists exactly the reference's - # codes and capability-constraints §4.2 its match bits); wai-live-web's functions for a - # page held to the same corpus natively, a page's log written through them verified; then - # the corpus's second implementation, stream_termination_verify.py, written from the two - # extensions' text and sharing no code with wai-rs, which recomputes every match and every - # log's outcome and facts and requires every mutated copy of the corpus to fail; standard - # library only, run isolated (-I). The browser's own run is joule-moq-web.yml's scenario - # termination. - - name: stream termination · wai-rs, wai-live-web and stream_termination_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib stream_termination::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test stream_termination) - (cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_stream_termination_vectors" verify ../live-conformance/termination) - (cd wai-live-web && cargo test --locked --features int_video --lib termination_tests) - python3 -I live-conformance/stream_termination_verify.py live-conformance/termination - # wai.live.world (extensions/live-worlds.md): the reference module's unit tests (the - # statement and its rules, the grant attribute) and tests/live_world.rs (vectors.json - # regenerates byte for byte from its fixed seeds and no other file is there, wai-rs gives - # every world its stated outcome, and §11 lists exactly the reference's codes, each of which - # a verifier raises the corpus gives); then the corpus's second implementation, - # live_worlds_verify.py, written from the extension's text and the sections it cites and - # sharing no code with wai-rs: the statement, each grant's signature and attributes, each - # receipt in its binary form under the owner's chain, each op's signature, owner, kind, - # Lamport rule and equivocation, and the session hash. Standard library only, run isolated. - - name: live worlds · wai-rs and live_worlds_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_world::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_world) - python3 -I live-conformance/live_worlds_verify.py live-conformance/worlds - # wai.live.e2ee (extensions/confidentiality.md): the corpus's second implementation, - # e2ee_verify.py, written from the extension's text and the documents it reads and - # sharing no code with the reference (joule-moq's secure_objects and joule-moq-mls, - # whose own checks run in joule-code-conformance.yml). Standard library only, run - # isolated (-I): AES, AES-GCM and RFC 9605's counter mode written from their - # specifications for test vectors only, HKDF and HMAC from the standard library, - # BLAKE3 and strict Ed25519 from claim-conformance. It checks every vector RFC 9605 - # and draft-ietf-moq-secure-objects-01 publish, every record, key schedule, sealed - # object and room of vectors.json (taking each epoch's application secret as given, - # §14), and requires every mutated copy of the corpus to fail. - - name: end-to-end encrypted rooms · e2ee_verify.py - working-directory: wai - shell: bash - run: python3 -I live-conformance/e2ee_verify.py live-conformance/e2ee - # wai.live.message and wai.live.call (extensions/live-messaging.md): tests/live_messaging.rs - # (vectors.json regenerates byte for byte from its fixed seeds and no other file is there, - # wai-rs gives every conversation, history request, call and record its stated outcome, and - # §12 lists exactly the reference's codes, each of which the corpus gives); then the corpus's - # second implementation, live_messaging_verify.py, written from the extension's text and - # sharing no code with wai-rs, which reads descriptors with live_rooms_verify.py and Key IDs, - # keys, AES-GCM and plaintexts with e2ee_verify.py, implements the receipts' binary form and - # every rule of the extension itself, verifies every signature with WAI's strict Ed25519 - # rule, requires every mutated copy of the corpus to fail, and drops each of its own checks - # in turn and requires the corpus to catch the weakened verifier; standard library only, run - # isolated (-I). - - name: live messaging · wai-rs and live_messaging_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_messaging) - python3 -I live-conformance/live_messaging_verify.py live-conformance/messaging - # wai.live.watermark (extensions/forensic-watermark.md): the reference module's unit tests - # (the CBOR reader's deterministic-encoding rules, tokens of both COSE structures and - # every mode read and their rules refuse, a record's rules, big-endian bit addressing, - # the exact match and the binding) and tests/live_watermark.rs (vectors.json regenerates - # byte for byte from its fixed seeds and no other file is there, wai-rs gives every - # token, session list, match and disclosure its stated outcome, and §11 lists exactly the - # reference's codes, each of which the corpus gives); then the corpus's second - # implementation, live_watermark_verify.py, written from the extension's text and sharing - # no code with wai-rs, which reads rooms with live_rooms_verify.py and tokens with a CBOR - # reader of its own, requires every mutated copy of the corpus to fail, and drops each of - # its own checks in turn and requires the corpus to catch the weakened verifier; standard - # library only, run isolated (-I). - - name: forensic watermark sessions · wai-rs and live_watermark_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib live_watermark::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test live_watermark) - python3 -I live-conformance/live_watermark_verify.py live-conformance/watermark - # wai.edge.derivation (extensions/deriving-edge.md): the reference module's unit tests - # (the operations pixel by pixel, which operations the registry's composition rule makes - # exact, the check each derivation takes, a record's reading rule by rule, its round trip - # and signature, the pinned output manifest, provenance items and AI-origin labels, the - # C2PA stores embedded in a still), the still writer and reader the module - # shares with wai-pack (png_still), and tests/derive_edge.rs (vectors.json regenerates - # byte for byte from its fixed seeds and no other file is there, wai-rs gives every vector - # its stated outcome, §9 lists exactly the reference's codes and §4 its derivations, and - # the reference edge wai_edge, run twice over a clip, makes a chain of two hops that - # verifies and is refused once a byte changes); then the corpus's second implementation, - # edge_verify.py, written from the extension's text and sharing no code with wai-rs, - # which reads receipts and writes stills itself, re-executes every replicate hop, - # requires every mutated copy of the corpus to fail, and drops each of its own checks in - # turn and requires the corpus to catch the weakened verifier; standard library only, - # run isolated (-I). - - name: deriving edge · wai-rs and edge_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib derive_edge::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib png_still::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test derive_edge) - python3 -I edge-conformance/edge_verify.py edge-conformance - # wai.live.injection (extensions/cue-injection.md) and Enhanced RTMP ingest - # (extensions/live-ingest.md §12): both corpora are generated through joule-moq-ingest, - # whose tests/conformance_corpus.rs regenerates them byte for byte - # (joule-code-conformance.yml). Here their second implementations, written from the - # texts and sharing no code with the reference: injection_verify.py translates every - # SCTE 104 message to its SCTE 35 sections and cues (the standard's sample among them, - # byte for byte), requires every refusal's result code and reason, runs the injector's - # rules over every scenario, verifies every chain with WAI's strict Ed25519 and derives - # its breaks, and refuses every negative chain at its event; ertmp_verify.py reads every - # RTMP message of every session and requires every object of every room track and what - # was not carried. Each then requires every mutated copy of its corpus to fail - # (--self-test). Standard library only, run isolated (-I). - - name: cue injection and Enhanced RTMP · injection_verify.py and ertmp_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - python3 -I live-conformance/injection_verify.py --self-test live-conformance/injection - python3 -I live-conformance/ertmp_verify.py --self-test live-conformance/ertmp - # The quantum corpus's second implementation (quantum-sim extension), integers only: - # it derives the circuit/1 phase table by its recurrence and the circuit2 table by an - # interval-certified procedure, and requires both literal tables and their digests, - # and the extension's Appendix B to state the same; - # rebuilds every file of both vector trees byte for byte and recomputes every manifest - # entry with its own reader, simulator, sampler and BLAKE3; refuses every reject file - # with its listed class and by the rule its name states; and requires dyadic_ladder - # to see every ±1 change of every P(k) under both tables. Standard library only, run - # isolated (-I), so no module from the working directory, the environment or the - # user's site-packages is imported. - - name: quantum referee.py - shell: bash - run: python3 -I wai/quantum-conformance/tools/referee.py wai/quantum-conformance - # chain_differential.py signs receipt chains around jwp-receipts §3 step 3 and §6.1 - # (two keys, two tracks, both payloads, replays, forks, gaps, the guard and the - # freshness bound), and chain_gap_differential.py genuine chains up to 90 groups - # delivered shuffled, withheld and walked back with rewritten links and forks; both - # require wai_jwp_vectors and jwp_verify.py to give every chain the same verdict. Two - # fixed seeds each, so a run is reproducible. - # Hybrid signatures (extensions/pq-signatures.md): the reference module's unit tests (its - # ML-DSA-65 against FIPS 204's ACVP vectors — keyGen, deterministic sigGen and sigVer — - # vendored in pq-conformance/acvp and pinned by SHA-256, and the composite against the - # draft's own vector, before anything is built on them), the synthesis reader's hybrid - # records, and tests/pq_signatures.rs (vectors.json regenerates byte for byte, every case - # reaches its verdict, §8 lists exactly the reference's codes, each given); then the - # corpus's second implementation, pq_verify.py, written from the extension's text and - # sharing no code with wai-rs, which verifies ML-DSA-65 from FIPS 204's text in pure - # Python (_mldsa.py), checks every ACVP verification vector and signature and the draft's - # vector, every case, the sizes §9 states and §8's table, requires every mutated case to be - # refused and the corpus to catch it with any one of its checks dropped; run isolated (-I). - # Last, the two give every case the same verdict. - - name: hybrid signatures · wai-rs and pq_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_PQ_FEATURES" --lib pq_sig::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_PQ_FEATURES" --lib synthesis::pq_tests) - (cd wai-rs && cargo test --no-default-features --features "$WAI_PQ_FEATURES" --test pq_signatures) - python3 -I pq-conformance/pq_verify.py pq-conformance - python3 -I pq-conformance/pq_verify.py verdicts pq-conformance/vectors.json > "$RUNNER_TEMP/pq-py.tsv" - (cd wai-rs && "$RUNNER_TEMP/wai-bins/pq/wai_pq_vectors" verdicts ../pq-conformance/vectors.json) > "$RUNNER_TEMP/pq-rs.tsv" - diff "$RUNNER_TEMP/pq-py.tsv" "$RUNNER_TEMP/pq-rs.tsv" - - name: claim_verify.py, differential.py, jwp_verify.py, chain_differential.py, chain_gap_differential.py, signer_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - python3 claim-conformance/claim_verify.py claim-conformance/vectors.json - python3 jwp-conformance/jwp_verify.py jwp-conformance/vectors.json - python3 signer-conformance/signer_verify.py signer-conformance/vectors.json - python3 signer-conformance/signer_verify.py verdicts signer-conformance/vectors.json > "$RUNNER_TEMP/signer-py.tsv" - # Each differential against both builds the first step copied (see its comment). - for build in union pq; do - bins="$RUNNER_TEMP/wai-bins/$build" - if [ "$build" = pq ]; then features=$WAI_PQ_FEATURES; else features=$WAI_FEATURES; fi - echo "::group::the differentials against the $build build (features $features)" - python3 claim-conformance/differential.py "$bins/wai_claim_vectors" --random 2000 --seed 1 - for seed in 1 2; do - python3 jwp-conformance/chain_differential.py "$bins/wai_jwp_vectors" --n 3000 --seed "$seed" - python3 jwp-conformance/chain_gap_differential.py "$bins/wai_jwp_vectors" --n 400 --seed "$seed" - done - "$bins/wai_signer_vectors" verdicts signer-conformance/vectors.json > "$RUNNER_TEMP/signer-rs-$build.tsv" - diff "$RUNNER_TEMP/signer-py.tsv" "$RUNNER_TEMP/signer-rs-$build.tsv" - echo "the $build build: every differential agrees" - echo "::endgroup::" - done - # Header probes (extensions/format-probes.md, projection.md): the reference module's unit - # tests (bits, Exp-Golomb and emulation prevention, the codes) and tests/format_probe.rs (the - # committed corpus regenerates byte for byte from its external vectors and holds no other - # generated file, every vector, binding and fact gets its stated outcome, and the - # extensions' tables list exactly the reference's probes and codes); then the five LCEVC - # streams fetch.sh fetches and checks by SHA-256, each of which wai_format_probe and the - # second implementation require (a missing one is a failure); then - # format_probe_verify.py, written from the two texts and sharing no code with wai-rs, - # which probes every vector itself and requires every mutated copy of the corpus to fail. - # Standard library only, run isolated (-I). - - name: header probes · wai-rs, the fetched LCEVC streams and format_probe_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib format_probe::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test format_probe) - fetched=$(mktemp -d) - bash format-probe-conformance/fetch.sh "$fetched" - (cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_format_probe" verify ../format-probe-conformance --fetched "$fetched") - python3 -I format-probe-conformance/format_probe_verify.py format-probe-conformance --fetched "$fetched" - # Media identification (extensions/media-identify.md): wai-rs's unit tests (hostile bytes - # through every walk, what a file reads as its bytes) and tests/identify.rs (the corpus - # regenerates byte for byte with every result; the extension's tables; `wai identify`); then - # identify_verify.py, written from the text and sharing no code with wai-rs but - # format_probe_verify.py's probes, which gives every file its whole result, holds ext/ to - # what an independent prober read of it, and requires every mutated copy to fail. - - name: media identification · wai-rs and identify_verify.py - working-directory: wai - shell: bash - run: | - set -euo pipefail - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib identify::) - (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test identify) - (cd wai-rs && "$RUNNER_TEMP/wai-bins/union/wai_identify_vectors" verify ../identify-conformance) - python3 -I identify-conformance/identify_verify.py identify-conformance - # CMSD reading (extensions/delivery-telemetry.md §6.6): the second implementation, written - # from that text and RFC 9651 with its own field parser, reads every field of - # telemetry-conformance/cmsd-read/ and every field expected/cmsd.json states, and requires - # every mutated copy to fail. wai-rs's reading is checked by wai_telemetry verify above. - - name: cmsd_read_verify.py - working-directory: wai - shell: bash - run: python3 -I telemetry-conformance/cmsd_read_verify.py telemetry-conformance - # LOC carriage (extensions/loc-carriage.md): the second implementation of joule-moq's loc - # module (whose tests/loc_vectors.rs runs in joule-code-conformance), written from the text, - # with its own MOQT -22 property decoder and BLAKE3 (claim-conformance/_blake3.py): it reads - # every object back to its envelope, writes it again byte for byte, gives every refused - # object and envelope its code, and requires every mutated copy, and the corpus under a - # copy of itself with a check dropped, to fail. - - name: loc_verify.py - working-directory: wai - shell: bash - run: python3 -I live-conformance/loc_verify.py live-conformance/loc + python3 ../../scripts/wai-conformance-lists.py prebuild > "$RUNNER_TEMP/wai-prebuild.sh" + cat "$RUNNER_TEMP/wai-prebuild.sh" + bash --noprofile --norc -eo pipefail "$RUNNER_TEMP/wai-prebuild.sh" + # Every step, in its order, each in a group of the log; the first that fails stops the + # job, as a failed step did, and is named in an error line with its file. + - name: Every verifier step (.github/wai-conformance/verifiers/) + shell: bash + run: python3 scripts/wai-conformance-lists.py run # Each feature list a step of the independent verifiers built alone, before those steps # shared one build of the union, still compiles alone: a feature that compiles only beside @@ -2870,22 +1066,17 @@ jobs: rustup toolchain install 1.99.1 --profile minimal rustup default 1.99.1 echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + command -v python3 >/dev/null 2>&1 || { SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo"; $SUDO apt-get update -qq; $SUDO apt-get install -y -qq python3; } - name: wai-rs, each feature list alone working-directory: wai/wai-rs shell: bash run: | set -euo pipefail - for f in key_discovery http,video_manifest,key_discovery stage_measure live_events live_room \ - capture_take live_compose live_record timed_text access_services live_metering energy_steering \ - live_moderation dred_exact live_ads live_provenance synthesis synthesis_mark neural_int \ - exact_upscale robustness ai_usage contribution settlement browser_sink browser_sink,receipt_claims \ - takedown influence direction co_performance age_assurance bearer_receipts carousel holdings exploration co_watch \ - telemetry_aggregate exact_tonemap splat_exact live_watermark derive_edge pq_signatures \ - synthesis,pq_signatures sink_energy key_release stitched_presentation prior_wire \ - prior_wire,neural_int scene_map avatar_face receipt_ledger stream_audit fleet native_sink \ - platform_encode remote_desktop stream_termination live_world format_probe live_messaging \ - conform_result revenue_share opportunity_report channel_lineup route_record; do + # The verifier steps' feature lists (`features:`, or `alone:` where a file gives one). + lists=$(python3 ../../scripts/wai-conformance-lists.py alone) + [ -n "$lists" ] || { echo "::error::no feature list was derived"; exit 1; } + for f in $lists; do echo "::group::$f" cargo check --no-default-features --features "$f" --tests echo "::endgroup::" diff --git a/CLAUDE.md b/CLAUDE.md index 274b5c24d..94257c377 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,6 +12,14 @@ This is the Transaction-Science **open-standards monorepo** (Apache-2.0 code + C Dev-machine push/upload specifics (LAN push-routing, credential, release-asset upload) are in the global operator notes (`~/.claude/CLAUDE.md`). +## Add files of your own; insert beside the entry you build on + +Many sessions' branches are open at once, and every line two of them append to is a conflict that serialises their merges. A change adds files of its own where the tree lets it (its changelog entry, and its lists in `wai-conformance.yml`), and inserts beside the entry it builds on in a file that stays shared: + +- **WAI's changelog:** one new file per change, `wai/changes/-.md`, holding the entry exactly as `wai/CHANGELOG.md` would ([`wai/changes/README.md`](wai/changes/README.md)). Never edit the head of `wai/CHANGELOG.md`; a release renders the files into it (`scripts/changelog-assemble.py --write`). The repository checks fail a pull request that changes `wai/`, adds or edits no entry file directly in `wai/changes/`, and has no commit message whose trailer block holds the line `Changelog: none`, exactly (a release, which renders the files and removes them, passes); one opened since the check reached main (the merge that brought it in) that adds its entry to the head of `wai/CHANGELOG.md` instead fails too. +- **`wai-conformance.yml`:** a new step of the independent verifiers is a file of `.github/wai-conformance/verifiers/`, a new cargo-class corpus a file of `.github/wai-conformance/corpora/`, and a new fast-integer-kernel test tags itself (`// wai-conformance: int-kernels`). The workflow's feature lists, test lists, steps and corpora, and the paths its gate runs on, are derived from them; add no line to the workflow for them, unless a step's `# paths:` names a file under none of the trigger's paths (`check` names it). `python3 scripts/wai-conformance-lists.py dry-run` prints what runs; `wai/CONTRIBUTING.md` states the header keys. +- **Files still shared** (`wai/wai-rs/Cargo.toml`'s features and targets, `[[test]]` entries included, `wai/wai-rs/src/lib.rs`'s `mod` lines, the tables of `codecs/registry.rs`, `wai/.gitattributes`, and the corpus table of `wai/live-conformance/README.md` with the verifier commands under it) are grouped by topic, not sorted: insert beside the entry yours builds on, never at the end, and never re-sort. A corpus whose bytes must never be converted gives its own directory a `.gitattributes` (`* -text`), as `wai/pin-conformance/` and `wai/live-conformance/scene/` do, rather than a line in `wai/.gitattributes`. Generated files (`wai/registry/capabilities.json`, `wai/REGISTRY.md`'s table, `wai/vocab/v2.*`) are sorted by the export: regenerate them, never merge them by hand. + ## Framing rules (public surfaces) - Describe every standard **on its own terms** — ingest open/private efforts into permissive-licensed Rust; **no competitive / SWOT framing**, no comparisons, no us-vs-them. Copy is self-referential only. diff --git a/scripts/changelog-assemble.py b/scripts/changelog-assemble.py new file mode 100755 index 000000000..44eddf055 --- /dev/null +++ b/scripts/changelog-assemble.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""changelog-assemble.py: the changelog's entries, one a file, rendered into its Unreleased +section for a release. + +A change adds its entry as a file of `wai/changes/` instead of editing the head of +`wai/CHANGELOG.md`, so that two changes' entries never edit the same line. +For a release this puts every entry at the head of `## Unreleased`, in a stable order, and +removes the files. + + changelog-assemble.py [--changelog wai/CHANGELOG.md] [--fragments wai/changes] + [--check | --write] + +With neither flag it prints the Unreleased section as it would be, changing nothing. +--check checks every file and exits 1 with one line a finding. --write writes the changelog +and removes the files it took. + +A file is named `-.md` (the date it was written and a few words of its +title, lowercase, joined by hyphens) and holds exactly one entry in the changelog's form: a +`### ` title line, a blank line, and the entry's text, with no other heading outside a code +block. The entries go newest date first and, within a date, by file name, each separated +from the next by one blank line; the same files give the same bytes. `README.md` is not an +entry. + +Standard library only. +""" +from __future__ import annotations + +import datetime +import os +import re +import sys + +NAME = re.compile(r"^(\d{4})-(\d{2})-(\d{2})-([a-z0-9]+(?:-[a-z0-9]+)*)\.md$") +UNRELEASED = "## Unreleased" + + +def fragment_files(directory): + """The entry files of `directory`, by name: every `*.md` but README.md.""" + if not os.path.isdir(directory): + return [] + return sorted(f for f in os.listdir(directory) if f.endswith(".md") and f != "README.md") + + +def check_fragment(name, text): + """Every finding about one entry file: its name and its form.""" + out = [] + m = NAME.match(name) + if not m: + out.append(f"{name}: not named -.md (a date, then lowercase words joined by hyphens)") + else: + try: + datetime.date(int(m.group(1)), int(m.group(2)), int(m.group(3))) + except ValueError: + out.append(f"{name}: {m.group(1)}-{m.group(2)}-{m.group(3)} is not a date") + if "\r" in text: + out.append(f"{name}: has carriage returns; lines end with \\n alone") + if not text.endswith("\n") or text.endswith("\n\n"): + out.append(f"{name}: does not end with exactly one newline") + lines = text.rstrip("\n").split("\n") + if not lines[0].startswith("### ") or not lines[0][4:].strip(): + out.append(f"{name}: its first line is not the entry's title (`### `)") + if len(lines) < 3 or lines[1] != "" or not "".join(lines[2:]).strip(): + out.append(f"{name}: the title is not followed by a blank line and the entry's text") + fence = False + for n, line in enumerate(lines[1:], 2): + if line.lstrip().startswith("```"): + fence = not fence + continue + if not fence and re.match(r"^#{1,6} ", line): + out.append(f"{name}: line {n}: a heading inside the entry; an entry is one `### ` title and its text") + if line != line.rstrip(): + out.append(f"{name}: line {n}: trailing white space") + if fence: + out.append(f"{name}: a code block is not closed") + return out + + +def order_key(name): + """Newest date first; within a date, by name.""" + m = NAME.match(name) + date = (int(m.group(1)), int(m.group(2)), int(m.group(3))) if m else (0, 0, 0) + return (tuple(-x for x in date), name) + + +def entries(directory): + """[(name, text)] of the entry files, in the order they are rendered, and the findings.""" + found, bad = [], [] + for name in fragment_files(directory): + with open(os.path.join(directory, name), encoding="utf-8", newline="") as fh: + text = fh.read() + bad += check_fragment(name, text) + found.append((name, text)) + found.sort(key=lambda e: order_key(e[0])) + return found, bad + + +def assemble(changelog_text, items): + """The changelog with the entries at the head of its Unreleased section.""" + lines = changelog_text.split("\n") + try: + at = lines.index(UNRELEASED) + except ValueError: + raise SystemExit(f"changelog-assemble: the changelog has no `{UNRELEASED}` line") + if not items: + return changelog_text + head = "\n".join(lines[: at + 1]) + rest = "\n".join(lines[at + 1:]).lstrip("\n") + block = "\n".join(text.rstrip("\n") + "\n" for _, text in items) + return head + "\n\n" + block + ("\n" + rest if rest else "") + + +def main(argv): + args = list(argv) + def opt(name, default): + if name in args: + i = args.index(name) + v = args[i + 1] + del args[i:i + 2] + return v + return default + changelog = opt("--changelog", "wai/CHANGELOG.md") + directory = opt("--fragments", "wai/changes") + mode = args[0] if args else "print" + if mode not in ("print", "--check", "--write") or len(args) > 1: + print(__doc__) + return 2 + items, bad = entries(directory) + if mode == "--check" or bad: + for b in bad: + print(f"{directory}/{b}") + if bad: + print(f"changelog-assemble: {len(bad)} finding(s); nothing was written") + return 1 + print(f"changelog-assemble: every entry under {directory}/ is one entry in the changelog's form.") + return 0 + with open(changelog, encoding="utf-8", newline="") as fh: + text = fh.read() + out = assemble(text, items) + if mode == "print": + lines = out.split("\n") + at = lines.index(UNRELEASED) + end = next((i for i in range(at + 1, len(lines)) if lines[i].startswith("## ")), len(lines)) + sys.stdout.write("\n".join(lines[at:end]).rstrip("\n") + "\n") + return 0 + with open(changelog, "w", encoding="utf-8", newline="") as fh: + fh.write(out) + for name, _ in items: + os.remove(os.path.join(directory, name)) + print(f"changelog-assemble: {len(items)} entr{'y' if len(items) == 1 else 'ies'} written to {changelog}, and their files removed") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/changelog-fragments-selftest.sh b/scripts/changelog-fragments-selftest.sh new file mode 100755 index 000000000..9f165e2b8 --- /dev/null +++ b/scripts/changelog-fragments-selftest.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# +# changelog-fragments-selftest.sh — the changelog's entry files are checked, and rendered the +# same way every time. +# +# scripts/check-changelog-fragment.py decides whether a pull request carries its changelog +# entry, and scripts/changelog-assemble.py renders the entry files into the changelog for a +# release. This builds a scratch repository laid out as this one is and requires: a change to +# wai/ with no entry fails, and passes with an entry file directly in wai/changes/ or with the +# trailer line `Changelog: none` (that line exactly, in a message's trailer block); an entry in +# a subdirectory, or a file there that is no entry, fails; a change outside wai/ needs none; a +# malformed entry fails, saying why; a pull request opened before the check landed passes as +# it is, with an edit of CHANGELOG.md or with nothing, and one opened since that adds its entry +# to the head of Unreleased fails, saying to add the file; the check lands when it reaches the +# base branch, by the merge, not when the commit that adds it was made, and before then every +# pull request counts as opened before it; a release (the entries rendered, their files +# removed) passes; commits the clone cannot compare, a shallow clone and an event payload with +# no creation time fail, saying why; and the assembler renders the files at the head of +# Unreleased in a stable order, the same bytes twice, writes them and removes the files, and +# refuses a malformed one. +set -uo pipefail + +# The job that runs this is a pull request's, whose own event payload must not decide here. +unset GITHUB_EVENT_PATH + +here=$(cd "$(dirname "$0")" && pwd) +check="$here/check-changelog-fragment.py" +asm="$here/changelog-assemble.py" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +R="$tmp/r" +mkdir -p "$R/scripts" "$R/wai/changes" "$R/wai/wai-rs" "$R/jcp" +cp "$check" "$asm" "$R/scripts/" +git -C "$R" init -q +git -C "$R" config user.email t@example.invalid +git -C "$R" config user.name t +printf '# changelog\n\nIntro.\n\n## Unreleased\n\n### The entry before\n\n- It was.\n\n## v0.1.0\n\n### Old\n\n- Old.\n' > "$R/wai/CHANGELOG.md" +printf '# Entries\n' > "$R/wai/changes/README.md" +echo a > "$R/wai/wai-rs/lib.rs"; echo b > "$R/jcp/SPEC.md" +# The commit that adds the check: it lands at 2026-11-04 12:00 UTC. +git -C "$R" add -A; GIT_COMMITTER_DATE="2026-11-04T12:00:00+00:00" git -C "$R" commit -qm base +BASE=$(git -C "$R" rev-parse HEAD) +# Event payloads of pull requests opened before and after it. +printf '{"pull_request": {"number": 1, "created_at": "2026-11-01T09:30:00Z"}}\n' > "$tmp/before.json" +printf '{"pull_request": {"number": 2, "created_at": "2026-11-04T08:00:01-04:00"}}\n' > "$tmp/after.json" +printf '{"pull_request": {"number": 3}}\n' > "$tmp/no-time.json" + +fails=0 +W="$R" # the repository the checks below run in +expect() { # expect <0|1> <what> <text the output must hold or ""> -- <command...> + local want=$1 what=$2 needle=$3 out rc + shift 4 + out=$(cd "$W" && "$@" 2>&1); rc=$? + if [ "$want" = 0 ] && [ $rc -ne 0 ]; then echo "FAIL $what: exit $rc"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi + if [ "$want" = 1 ] && [ $rc -ne 1 ]; then echo "FAIL $what: exit $rc, not 1"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi + if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi + echo "ok $what" +} +branch() { git -C "$R" checkout -q -B "$1" "$BASE"; } +commit() { git -C "$R" add -A; git -C "$R" commit -q -m "${1:-change}"; git -C "$R" rev-parse HEAD; } +entry='### A thing changed\n\n- **What.** It changed.\n' +C="python3 -I scripts/check-changelog-fragment.py" +opened() { env GITHUB_EVENT_PATH="$tmp/$1.json" python3 -I scripts/check-changelog-fragment.py "${@:2}"; } # opened <before|after|no-time> <args> +by_hand() { perl -pi -e 's/^### The entry before$/### A thing changed\n\n- It did.\n\n### The entry before/' "${1:-$R}/wai/CHANGELOG.md"; } # by_hand [<repository>] + +# The pull request check. +branch outside; echo c > "$R/jcp/SPEC.md"; H=$(commit) +expect 0 "a change outside wai/ needs no entry" "touches nothing under wai/" -- $C "$BASE" "$H" +branch bare; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit) +expect 1 "a change to wai/ with no entry fails" "adds no entry" -- $C "$BASE" "$H" +printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit) +expect 0 "and passes once it adds an entry file" "2026-11-04-a-thing.md" -- $C "$BASE" "$H" +expect 0 "an entry file passes whenever the pull request was opened" "2026-11-04-a-thing.md" -- opened after "$BASE" "$H" + +# Only directly in wai/changes/, where the assembler and wai breaking read. +branch subdir; echo c > "$R/wai/wai-rs/lib.rs"; mkdir -p "$R/wai/changes/sub"; printf "$entry" > "$R/wai/changes/sub/2026-11-04-planted.md"; H=$(commit) +expect 1 "an entry file in a subdirectory of wai/changes/ fails" "wai/changes/sub/2026-11-04-planted.md: an entry file goes directly in wai/changes/, not in a subdirectory" -- $C "$BASE" "$H" +printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit) +expect 1 "and fails beside a good entry too, which the subdirectory's would never join" "not in a subdirectory" -- $C "$BASE" "$H" +branch notmd; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.txt"; H=$(commit) +expect 1 "a file of wai/changes/ that is no .md entry fails" "not an entry file" -- $C "$BASE" "$H" + +# `Changelog: none`: that line exactly, in the trailer block of a commit message. +branch none; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'lockfile only\n\nChangelog: none') +expect 0 "a commit that says Changelog: none passes" "Changelog: none" -- $C "$BASE" "$H" +branch none-trailers; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'lockfile only\n\nWhy, in a paragraph.\n\nChangelog: none\nCo-Authored-By: T <t@example.invalid>') +expect 0 "among the other trailers too" "Changelog: none" -- $C "$BASE" "$H" +branch none-in-text; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'the word Changelog: none mid-line does not count') +expect 1 "the trailer starts its line" "adds no entry" -- $C "$BASE" "$H" +branch none-of; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nChangelog: none of the tests changed') +expect 1 "'Changelog: none of ...' is not the trailer" "adds no entry" -- $C "$BASE" "$H" +branch none-case; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nchangelog: None') +expect 1 "nor is it in another case" "adds no entry" -- $C "$BASE" "$H" +branch none-mid; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nChangelog: none\n\nA paragraph after it.') +expect 1 "a trailer line before the last paragraph is no trailer" "adds no entry" -- $C "$BASE" "$H" +branch none-body; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nA body line, then no blank line.\nChangelog: none') +expect 1 "nor is one in a paragraph of prose" "in its last paragraph, after a blank line" -- $C "$BASE" "$H" +branch none-subject; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit 'Changelog: none') +expect 1 "nor is a subject line" "adds no entry" -- $C "$BASE" "$H" + +branch readme; echo c > "$R/wai/wai-rs/lib.rs"; echo more >> "$R/wai/changes/README.md"; H=$(commit) +expect 1 "the entries' README is no entry" "adds no entry" -- $C "$BASE" "$H" +branch badname; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry" > "$R/wai/changes/A Thing.md"; H=$(commit) +expect 1 "an entry file named otherwise fails" "not named <yyyy-mm-dd>-<slug>.md" -- $C "$BASE" "$H" +branch baddate; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry" > "$R/wai/changes/2026-02-30-a-thing.md"; H=$(commit) +expect 1 "an entry file whose date is no date fails" "is not a date" -- $C "$BASE" "$H" +branch notitle; echo c > "$R/wai/wai-rs/lib.rs"; printf -- '- **What.** It changed.\n' > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit) +expect 1 "an entry with no title fails" "is not the entry's title" -- $C "$BASE" "$H" +branch two; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry\n### Another\n\n- And.\n" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit) +expect 1 "a file holding two entries fails" "a heading inside the entry" -- $C "$BASE" "$H" +branch fence; echo c > "$R/wai/wai-rs/lib.rs"; printf '### Code\n\n- Run:\n\n```bash\n# a comment, not a heading\nwai hello\n```\n' > "$R/wai/changes/2026-11-04-code.md"; H=$(commit) +expect 0 "a # line inside a code block is no heading" "2026-11-04-code.md" -- $C "$BASE" "$H" + +# The transition: opened before the check landed, a pull request passes as it is. +branch old-style; echo c > "$R/wai/wai-rs/lib.rs"; by_hand; H=$(commit) +expect 0 "opened before the check landed, an edit of CHANGELOG.md's head passes" "before this check landed" -- opened before "$BASE" "$H" +expect 1 "opened since, the same edit fails, saying to add the entry file" "move each to wai/changes/<yyyy-mm-dd>-<slug>.md" -- opened after "$BASE" "$H" +expect 1 "and it names the entry it found" "(\`### A thing changed\`)" -- opened after "$BASE" "$H" +expect 1 "with no event payload, a change is taken as opened now" "taken as opened now" -- $C "$BASE" "$H" +branch bare-old; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit) +expect 0 "opened before, a change with no changelog edit at all passes too" "passes as it is" -- opened before "$BASE" "$H" +expect 1 "and fails when opened since" "adds no entry" -- opened after "$BASE" "$H" +branch both; echo c > "$R/wai/wai-rs/lib.rs"; by_hand; printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit) +expect 1 "opened since, an entry file does not excuse an entry added to the head as well" "to the head of \`## Unreleased\`" -- opened after "$BASE" "$H" +expect 1 "an event payload with no creation time fails, saying so" "cannot be read from the event payload" -- opened no-time "$BASE" "$(git -C "$R" rev-parse bare-old)" + +# The check lands when it reaches the base branch: the first commit of main's first-parent +# history whose tree holds it, here the merge of the branch that adds it, at that merge's own +# time. A second repository, whose main starts without the check: the branch that adds it is +# committed on 11-05 at 03:09 and merged on 11-06 at 10:00; a pull request opened on 11-05 at +# 12:00, between the two, edits the head of Unreleased and then merges main in. +T="$tmp/t" +mkdir -p "$T/wai/changes" "$T/wai/wai-rs" +git -C "$T" init -q; git -C "$T" symbolic-ref HEAD refs/heads/main +git -C "$T" config user.email t@example.invalid; git -C "$T" config user.name t +cp "$R/wai/changes/README.md" "$T/wai/changes/" +printf '# changelog\n\nIntro.\n\n## Unreleased\n\n### The entry before\n\n- It was.\n' > "$T/wai/CHANGELOG.md" +echo a > "$T/wai/wai-rs/lib.rs" +at() { local d=$1; shift; GIT_AUTHOR_DATE="$d" GIT_COMMITTER_DATE="$d" git -C "$T" "$@"; } # at <time> <git args> +tcommit() { git -C "$T" add -A; at "$1" commit -q -m "$2"; git -C "$T" rev-parse HEAD; } # tcommit <time> <message> +M0=$(tcommit 2026-11-04T00:00:00+00:00 "main, before the check") +git -C "$T" checkout -q -b adds-check; mkdir -p "$T/scripts"; cp "$check" "$asm" "$T/scripts/" +A=$(tcommit 2026-11-05T03:09:34+00:00 "adds the check") +git -C "$T" checkout -q -b early "$M0"; echo e > "$T/wai/wai-rs/lib.rs"; tcommit 2026-11-05T04:00:00+00:00 "on main before the check" >/dev/null +git -C "$T" checkout -q -b on-check "$A"; echo q > "$T/wai/wai-rs/lib.rs"; by_hand "$T"; Q=$(tcommit 2026-11-05T13:00:00+00:00 "built on the check's branch") +git -C "$T" checkout -q -b window "$M0"; echo w > "$T/wai/wai-rs/lib.rs"; by_hand "$T"; tcommit 2026-11-05T12:00:00+00:00 "opened between" >/dev/null +git -C "$T" checkout -q -b window-bare "$M0"; echo v > "$T/wai/wai-rs/lib.rs"; tcommit 2026-11-05T12:00:00+00:00 "opened between, no entry" >/dev/null +git -C "$T" checkout -q main; at 2026-11-06T10:00:00+00:00 merge -q --no-ff -m "merge the check" adds-check; M1=$(git -C "$T" rev-parse HEAD) +for b in window window-bare; do git -C "$T" checkout -q "$b"; at 2026-11-06T11:00:00+00:00 merge -q --no-ff -m "merge main" main; done +P1=$(git -C "$T" rev-parse window); P2=$(git -C "$T" rev-parse window-bare) +git -C "$T" checkout -q -b after main; echo f > "$T/wai/wai-rs/lib.rs"; by_hand "$T"; P3=$(tcommit 2026-11-06T12:30:00+00:00 "opened after the merge") +git -C "$T" checkout -q main +printf '{"pull_request": {"number": 4, "created_at": "2026-11-05T12:00:00Z"}}\n' > "$tmp/window.json" +printf '{"pull_request": {"number": 5, "created_at": "2026-11-05T13:00:00Z"}}\n' > "$tmp/on-check.json" +printf '{"pull_request": {"number": 6, "created_at": "2026-11-06T12:00:00Z"}}\n' > "$tmp/merged.json" +W="$T" +expect 0 "opened after the commit that adds the check but before its merge, an edit of the head passes" \ + "before this check landed on the base branch (2026-11-06 10:00:00 UTC, ${M1:0:12})" -- opened window "$M1" "$P1" +expect 0 "and so does one with no entry at all" "passes as it is" -- opened window "$M1" "$P2" +expect 1 "opened after the merge, an edit of the head fails" \ + "Since this check landed on the base branch (2026-11-06 10:00:00 UTC, ${M1:0:12}; this pull request: 2026-11-06 12:00:00 UTC)" -- opened merged "$M1" "$P3" +expect 0 "before the check is on the base branch, a pull request counts as opened before it landed" \ + "this check is not on the base branch yet" -- opened on-check "$M0" "$Q" +expect 0 "and so does a local run, with no event payload" "not on the base branch yet" -- $C "$M0" "$Q" +S3="$tmp/shallow3"; git clone -q --no-local --depth 2 --branch early "file://$T" "$S3" 2>/dev/null +out=$(cd "$S3" && env GITHUB_EVENT_PATH="$tmp/window.json" python3 -I "$check" "$M0" "$(git -C "$T" rev-parse early)" 2>&1); rc=$? +if [ $rc -eq 1 ] && grep -qF "this shallow clone's history stops at ${M0:0:12} before any commit of it holds scripts/check-changelog-fragment.py" <<<"$out"; then + echo "ok a shallow clone whose base's history stops before the check fails, saying so" +else echo "FAIL a shallow clone with the base's history cut short: exit $rc"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); fi +W="$R" + +# A release renders the entries into the changelog and removes their files. +branch release; printf '### First\n\n- A.\n' > "$R/wai/changes/2026-11-04-first.md"; printf '### Second\n\n- B.\n' > "$R/wai/changes/2026-11-03-second.md"; P=$(commit) +(cd "$R" && python3 -I scripts/changelog-assemble.py --write >/dev/null); H=$(commit "release") +expect 0 "a release (entries rendered, their files removed) passes" "renders 2 entries of wai/changes/ into wai/CHANGELOG.md" -- opened after "$P" "$H" +git -C "$R" checkout -q release; perl -pi -e 's/^### The entry before$/### Slipped in\n\n- By hand.\n\n### The entry before/' "$R/wai/CHANGELOG.md"; H2=$(commit "release, and one by hand") +expect 1 "but an entry it adds by hand beside them fails" "(\`### Slipped in\`)" -- opened after "$P" "$H2" +branch unrendered; printf '### First\n\n- A.\n' > "$R/wai/changes/2026-11-04-first.md"; P=$(commit) +rm "$R/wai/changes/2026-11-04-first.md"; perl -pi -e 's/^Intro\.$/Intro, reworded./' "$R/wai/CHANGELOG.md"; H=$(commit "drops an entry") +expect 1 "and removing entry files beside an edit that does not render them is no release" "adds no entry" -- opened after "$P" "$H" + +# It never passes unchecked. +expect 1 "commits the clone cannot compare fail, saying so" "cannot be compared" -- $C "$BASE" 0000000000000000000000000000000000000000 +S1="$tmp/shallow1"; git clone -q --no-local --depth 1 --branch bare-old "file://$R" "$S1" 2>/dev/null; git -C "$S1" remote remove origin +out=$(cd "$S1" && python3 -I scripts/check-changelog-fragment.py "$BASE" "$(git -C "$R" rev-parse bare-old)" 2>&1); rc=$? +if [ $rc -eq 1 ] && grep -qF "cannot be compared in this clone, even after fetching its history" <<<"$out"; then echo "ok a shallow clone that cannot fetch the base fails, saying so" +else echo "FAIL a shallow clone with no way to the base: exit $rc"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); fi +S2="$tmp/shallow2"; git clone -q --no-local --depth 2 --branch bare-old "file://$R" "$S2" 2>/dev/null +out=$(cd "$S2" && env GITHUB_EVENT_PATH="$tmp/before.json" python3 -I scripts/check-changelog-fragment.py "$BASE" "$(git -C "$R" rev-parse bare-old)" 2>&1); rc=$? +if [ $rc -eq 1 ] && grep -qF "is where this shallow clone's history stops, so the time this check landed is unknown" <<<"$out"; then echo "ok a shallow clone cannot tell when the check landed, and fails, saying so" +else echo "FAIL a shallow clone and the transition: exit $rc"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); fi + +# The assembler. +branch assemble +printf '### Second of the day\n\n- B.\n' > "$R/wai/changes/2026-11-04-b-second.md" +printf '### First of the day\n\n- A.\n' > "$R/wai/changes/2026-11-04-a-first.md" +printf '### Older\n\n- Before.\n' > "$R/wai/changes/2026-10-30-older.md" +want=$(printf '## Unreleased\n\n### First of the day\n\n- A.\n\n### Second of the day\n\n- B.\n\n### Older\n\n- Before.\n\n### The entry before\n\n- It was.\n') +got=$(cd "$R" && python3 -I scripts/changelog-assemble.py); again=$(cd "$R" && python3 -I scripts/changelog-assemble.py) +if [ "$got" = "$want" ] && [ "$again" = "$got" ]; then echo "ok entries render newest date first, by name within a date, the same twice" +else echo "FAIL the rendered section"; diff <(printf '%s\n' "$want") <(printf '%s\n' "$got") | sed 's/^/ /'; fails=$((fails+1)); fi +expect 0 "--check passes well-formed entries" "one entry in the changelog's form" -- python3 -I scripts/changelog-assemble.py --check +expect 0 "--write writes the changelog" "3 entries written" -- python3 -I scripts/changelog-assemble.py --write +full=$(printf '# changelog\n\nIntro.\n\n%s\n\n## v0.1.0\n\n### Old\n\n- Old.\n' "$want") +if [ "$(cat "$R/wai/CHANGELOG.md")" = "$full" ]; then echo "ok and the changelog is the rendered section in place, the rest unchanged" +else echo "FAIL the written changelog"; diff <(printf '%s\n' "$full") "$R/wai/CHANGELOG.md" | sed 's/^/ /'; fails=$((fails+1)); fi +left=$(cd "$R/wai/changes" && ls) +[ "$left" = "README.md" ] && echo "ok and the entry files are removed, the README kept" || { echo "FAIL files left: $left"; fails=$((fails+1)); } +expect 0 "with no entry files, it changes nothing" "" -- python3 -I scripts/changelog-assemble.py --write +[ "$(cat "$R/wai/CHANGELOG.md")" = "$full" ] && echo "ok and the changelog stays as it was" || { echo "FAIL the changelog changed with no entries"; fails=$((fails+1)); } +printf '### Trailing\n\n- Space. \n' > "$R/wai/changes/2026-11-04-trailing.md" +expect 1 "a malformed entry fails --check" "trailing white space" -- python3 -I scripts/changelog-assemble.py --check +expect 1 "and --write writes nothing" "nothing was written" -- python3 -I scripts/changelog-assemble.py --write +[ -f "$R/wai/changes/2026-11-04-trailing.md" ] && [ "$(cat "$R/wai/CHANGELOG.md")" = "$full" ] \ + && echo "ok the file and the changelog are left as they were" || { echo "FAIL a refused --write changed something"; fails=$((fails+1)); } +rm "$R/wai/changes/2026-11-04-trailing.md" +printf '### No newline\n\n- At the end.' > "$R/wai/changes/2026-11-04-end.md" +expect 1 "an entry with no newline at its end fails" "exactly one newline" -- python3 -I scripts/changelog-assemble.py --check + +if [ $fails -ne 0 ]; then echo "changelog-fragments-selftest: $fails failure(s)"; exit 1; fi +echo "changelog-fragments-selftest: the check and the assembler pass what they should and fail each way they should." diff --git a/scripts/check-changelog-fragment.py b/scripts/check-changelog-fragment.py new file mode 100755 index 000000000..dc20abb16 --- /dev/null +++ b/scripts/check-changelog-fragment.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""check-changelog-fragment.py: a pull request that changes WAI says what it changed, in an +entry of its own. + + check-changelog-fragment.py <base-sha> <head-sha> + +A pull request whose diff (`git diff base...head`) changes anything under `wai/` must add or +edit an entry file directly in `wai/changes/` (one entry a file; see wai/changes/README.md), +in the changelog's form (scripts/changelog-assemble.py). A file in a subdirectory of +`wai/changes/` is refused: the assembler and `wai breaking` read only the directory itself, +so such an entry would never reach the changelog. + +A change with nothing for the changelog to say (a lockfile bump, a fix to a test) says so +with the trailer line `Changelog: none`, exactly, in the trailer block of one of its commit +messages (the message's last paragraph, after a blank line, every line of it a `Key: value` +trailer), and passes. + +A release's change, which renders the entry files into `wai/CHANGELOG.md` and removes them +(`changelog-assemble.py --write`), passes without one. + +The transition. A pull request opened before this check landed passes as it is: it may edit +the head of `wai/CHANGELOG.md` instead, or carry no entry at all. The time it was opened is +the pull request's `created_at` in the event payload ($GITHUB_EVENT_PATH). The time the check +landed is when it reached the base branch: the committer time of the first commit on the +first-parent history of <base-sha> (the pull request's base, the tip of main) whose tree +holds this file. When the check comes in by a merge commit, that is the merge's own time, +not the time of the branch commit that added the file. Until the check is on the base branch +at all, every pull request counts as opened before it landed. One opened since it landed +that adds an entry to the head of `## Unreleased` in `wai/CHANGELOG.md` fails, and is told +to add the entry as a file of `wai/changes/` instead. With no event payload (a local run), +the change is taken as opened now. + +Exit 0 when the change carries its entry, says it needs none, is a release's, was opened +before the check landed, or touches nothing under wai/; 1 otherwise, saying what to do. The +check never passes unchecked: a clone too shallow to diff is deepened first, and when the +two commits still cannot be compared, or git fails, or the times above cannot be read (a +shallow clone whose first-parent history stops before the answer is known), it fails and +says why (the workflow checks out the whole history, so this is a fault). + +Standard library only; runs on Python 3.9. +""" +from __future__ import annotations + +import collections +import datetime +import importlib.util +import json +import os +import posixpath +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +WAI = "wai/" +CHANGELOG = "wai/CHANGELOG.md" +FRAGMENTS = "wai/changes" +README = "wai/changes/README.md" +SELF = "scripts/" + os.path.basename(__file__) +UNRELEASED = "## Unreleased" +TRAILER = "Changelog: none" +TRAILER_LINE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]*:\s") +ISO = re.compile(r"^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(Z|[+-]\d{2}:?\d{2})$") + + +class Fault(Exception): + """Something the check needs and cannot read: it fails, saying what.""" + + +def git(*args, cwd=None): + return subprocess.run(["git", *args], capture_output=True, text=True, cwd=cwd) + + +def must(*args, cwd=None): + r = git(*args, cwd=cwd) + if r.returncode != 0: + raise Fault(f"`git {' '.join(args)}` failed: {r.stderr.strip() or 'exit ' + str(r.returncode)}") + return r.stdout + + +def assembler(): + spec = importlib.util.spec_from_file_location("changelog_assemble", os.path.join(HERE, "changelog-assemble.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def comparable(base, head): + """The merge base of base and head, fetching history once if the clone lacks it.""" + mb = git("merge-base", base, head) + if mb.returncode == 0: + return mb.stdout.strip() + if git("rev-parse", "--is-shallow-repository").stdout.strip() == "true": + git("fetch", "-q", "--no-tags", "--unshallow", "origin") + for sha in (base, head): + if git("cat-file", "-e", f"{sha}^{{commit}}").returncode != 0: + git("fetch", "-q", "--no-tags", "origin", sha) + mb = git("merge-base", base, head) + return mb.stdout.strip() if mb.returncode == 0 else None + + +def trailers(message): + """The lines of a commit message's trailer block: its last paragraph, when that is not + the subject and every line of it is a `Key: value` trailer (or a continuation line of + one, which starts with white space); otherwise none.""" + paras = [p for p in re.split(r"\n[ \t]*\n", message.strip("\n")) if p.strip()] + if len(paras) < 2: + return [] + out = [] + for line in paras[-1].split("\n"): + if line[:1] in (" ", "\t") and out: + continue + if not TRAILER_LINE.match(line): + return [] + out.append(line.rstrip()) + return out + + +def says_none(mb, head): + log = must("log", "--format=%B%x00", f"{mb}..{head}") + return any(TRAILER in trailers(m) for m in log.split("\x00")) + + +def epoch(text, what): + m = ISO.match(text or "") + if not m: + raise Fault(f"{what} `{text}` is not an ISO 8601 time with its zone") + y, mo, d, h, mi, s, z = m.groups() + if z == "Z": + tz = datetime.timezone.utc + else: + z = z.replace(":", "") + sign = 1 if z[0] == "+" else -1 + tz = datetime.timezone(sign * datetime.timedelta(hours=int(z[1:3]), minutes=int(z[3:5]))) + return int(datetime.datetime(int(y), int(mo), int(d), int(h), int(mi), int(s), tzinfo=tz).timestamp()) + + +def when(t): + return datetime.datetime.fromtimestamp(t, datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + +def landed(base): + """(commit, its committer time) of the first commit on the first-parent history of base + whose tree holds this file: when the check reached the base branch. None when no commit + of that history holds it: the check is not on the base branch yet. A shallow clone's + history stops at a commit whose parents it lacks, so when that commit already holds the + file, or the file is found nowhere before it, the answer is unknown and the check fails.""" + out = must("log", "--first-parent", "--reverse", "--format=%H %ct", base) + rows = [l.split() for l in out.split("\n") if l.strip()] + if not rows or any(len(r) != 2 for r in rows): + raise Fault(f"`git log --first-parent {base}` listed no commits, so the time this check landed is unknown") + query = "".join(f"{sha}:{SELF}\n" for sha, _ in rows) + r = subprocess.run(["git", "cat-file", "--batch-check"], input=query, capture_output=True, text=True) + held = r.stdout.split("\n")[:len(rows)] + if r.returncode != 0 or len(held) != len(rows) or any(not l.strip() for l in held): + raise Fault(f"`git cat-file --batch-check` could not say which commits of {base}'s first-parent history hold {SELF}: " + f"{r.stderr.strip() or 'exit ' + str(r.returncode)}") + first = next((i for i, l in enumerate(held) if not l.endswith(" missing")), None) + shallow = must("rev-parse", "--git-path", "shallow").strip() + if os.path.isfile(shallow): + with open(shallow, encoding="utf-8") as fh: + stops = set(fh.read().split()) + oldest = rows[0][0] + if oldest in stops and first in (None, 0): + found = (f"the first commit of it that holds {SELF}, {oldest[:12]}, is where this shallow clone's history stops" + if first == 0 else f"this shallow clone's history stops at {oldest[:12]} before any commit of it holds {SELF}") + raise Fault(f"on the first-parent history of {base[:12]}, {found}, so the time this check landed is unknown; " + f"check out the whole history (fetch-depth: 0)") + if first is None: + return None + sha, t = rows[first] + return sha, int(t) + + +def opened(): + """The pull request's creation time, from the event payload, or None with no payload.""" + path = os.environ.get("GITHUB_EVENT_PATH") + if not path: + return None + try: + with open(path, encoding="utf-8") as fh: + event = json.load(fh) + created = event["pull_request"]["created_at"] + except (OSError, ValueError, KeyError, TypeError) as e: + raise Fault(f"the pull request's creation time cannot be read from the event payload {path} ({type(e).__name__}: {e})") + return epoch(created, "the pull request's created_at") + + +def unreleased_titles(text): + """The `### ` titles of the changelog's Unreleased section.""" + lines = text.split("\n") + if UNRELEASED not in lines: + return [] + at = lines.index(UNRELEASED) + end = next((i for i in range(at + 1, len(lines)) if lines[i].startswith("## ")), len(lines)) + return [l for l in lines[at + 1:end] if l.startswith("### ")] + + +def show(rev, path): + r = git("show", f"{rev}:{path}") + return r.stdout if r.returncode == 0 else "" + + +def check(base, head): + mb = comparable(base, head) + if mb is None: + raise Fault(f"{base} and {head} cannot be compared in this clone, even after fetching its history; the entry was not checked") + d = must("diff", "--name-status", "--no-renames", f"{mb}..{head}") + changed = [l.split("\t", 1) for l in d.splitlines() if "\t" in l] + wai = [p for _, p in changed if p.startswith(WAI)] + if not wai: + print("changelog: this change touches nothing under wai/, so it needs no entry") + return 0 + + bad, written, removed = [], [], [] + for s, p in changed: + if not p.startswith(FRAGMENTS + "/") or p == README: + continue + if s == "D": + if posixpath.dirname(p) == FRAGMENTS and p.endswith(".md"): + removed.append(p) + elif posixpath.dirname(p) != FRAGMENTS: + bad.append(f"{p}: an entry file goes directly in {FRAGMENTS}/, not in a subdirectory: " + f"the assembler and `wai breaking` read only {FRAGMENTS}/ itself, so this entry would never reach the changelog") + elif not p.endswith(".md"): + bad.append(f"{p}: not an entry file: an entry is {FRAGMENTS}/<yyyy-mm-dd>-<slug>.md") + else: + written.append(p) + if written: + asm = assembler() + for p in written: + bad += [f"{FRAGMENTS}/{b}" for b in asm.check_fragment(posixpath.basename(p), must("show", f"{head}:{p}"))] + + # Entries added to the head of Unreleased by hand, rather than rendered there from the + # files this change removes (a release). + by_hand, release = [], False + if any(p == CHANGELOG for _, p in changed): + after = show(head, CHANGELOG) + new = collections.Counter(unreleased_titles(after)) - collections.Counter(unreleased_titles(show(mb, CHANGELOG))) + titles = [show(mb, p).split("\n", 1)[0] for p in removed] + by_hand = sorted((new - collections.Counter(titles)).elements()) + release = bool(titles) and all(t in after.split("\n") for t in titles) + + state = {} + def before(): + """Whether the pull request was opened before this check landed on the base branch + (read once). Until it is on the base branch at all, every pull request was.""" + if "before" not in state: + at = landed(base) + o = opened() + state["opened"] = "no event payload: taken as opened now" if o is None else when(o) + if at is None: + state["landed"] = None + state["before"] = True + else: + sha, t = at + state["landed"] = f"{when(t)}, {sha[:12]}" + state["before"] = o is not None and o < t + return state["before"] + + if by_hand and not before(): + titles = "; ".join(f"`{t}`" for t in by_hand[:3]) + ("; ..." if len(by_hand) > 3 else "") + bad.append(f"changelog: this change adds {len(by_hand)} entr{'y' if len(by_hand) == 1 else 'ies'} to the head of " + f"`{UNRELEASED}` in {CHANGELOG} ({titles}). Since this check landed on the base branch ({state['landed']}; this pull request: " + f"{state['opened']}), a change adds its entry as a file instead: move each to " + f"{FRAGMENTS}/<yyyy-mm-dd>-<slug>.md, holding it exactly as the changelog would, and leave {CHANGELOG} as it was " + f"(see {README}).") + if bad: + for b in bad: + print(f"::error::{b}") + return 1 + if written: + print(f"changelog: this change's entry is {', '.join(written)}") + return 0 + if release: + print(f"changelog: this change renders {len(removed)} entr{'y' if len(removed) == 1 else 'ies'} of {FRAGMENTS}/ into {CHANGELOG} (a release)") + return 0 + if says_none(mb, head): + print(f"changelog: a commit of this change says `{TRAILER}` in its trailers") + return 0 + if before(): + if state["landed"] is None: + print(f"changelog: this check is not on the base branch yet (no commit of the first-parent history of " + f"{base[:12]} holds {SELF}), so this pull request counts as opened before it landed and passes as it is; " + f"one opened after it lands adds its entry as a file of {FRAGMENTS}/") + else: + print(f"changelog: this pull request was opened ({state['opened']}) before this check landed on the base branch " + f"({state['landed']}), so it passes as it is; one opened since adds its entry as a file of {FRAGMENTS}/") + return 0 + print(f"::error::changelog: this change touches {len(wai)} file(s) under wai/ ({', '.join(wai[:3])}{', ...' if len(wai) > 3 else ''}) " + f"and adds no entry. Add one file, {FRAGMENTS}/<yyyy-mm-dd>-<slug>.md, holding the entry as the changelog " + f"would (a `### ` title, a blank line, its text; see {README}), or, if there is nothing for the changelog " + f"to say, end a commit message with the trailer line `{TRAILER}` (in its last paragraph, after a blank line, " + f"with the other trailers).") + return 1 + + +def main(argv): + if len(argv) != 2 or not argv[0] or not argv[1]: + print("usage: check-changelog-fragment.py <base-sha> <head-sha>") + return 2 + try: + return check(*argv) + except Fault as e: + print(f"::error::changelog: {e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/check-comparative-prose.sh b/scripts/check-comparative-prose.sh index 0219e5162..c6e7e6a0b 100755 --- a/scripts/check-comparative-prose.sh +++ b/scripts/check-comparative-prose.sh @@ -9,8 +9,8 @@ # # Scope: the same files as scripts/check-public-names.sh — the tracked files # under wai/ with the extensions .md .rs .toml .py .sh .mjs .js .json .html -# .css .ttl .h .c .txt and wai/'s .gitignore files, except wai/CHANGELOG.md: -# its history is a record, and only names are redacted from it. Files given as arguments replace that +# .css .ttl .h .c .txt and wai/'s .gitignore files, except wai/CHANGELOG.md and its entries +# waiting under wai/changes/ (scripts/changelog-assemble.py): its history is a record, and only names are redacted from it. Files given as arguments replace that # list (the self-test, scripts/test-public-text-checks.sh, uses this). # # Opt-out, for a functional identifier or a quotation only (an environment @@ -78,7 +78,7 @@ if [ $# -gt 0 ]; then else while IFS= read -r f; do case "$f" in - wai/CHANGELOG.md) continue ;; + wai/CHANGELOG.md|wai/changes/[0-9]*.md) continue ;; esac files+=("$f") done < <(git ls-files 'wai/*.md' 'wai/*.rs' 'wai/*.toml' 'wai/*.py' 'wai/*.sh' 'wai/*.mjs' 'wai/*.js' \ diff --git a/scripts/check-no-vanity-metrics.sh b/scripts/check-no-vanity-metrics.sh index d0278d0a6..328546987 100755 --- a/scripts/check-no-vanity-metrics.sh +++ b/scripts/check-no-vanity-metrics.sh @@ -46,14 +46,15 @@ cd "$REPO_ROOT" PATTERN='(^|[^[:alnum:]_-])[0-9][0-9,]*[+kKMG]?[+ ]*([Cc]rates?|[Tt]ests?|LOC|loc|[Ss]amplers?|[Aa]dapters?|[Bb]ackends?|[Ii]ntegrations?)([ .,;:!?)$]|$)' # Build the file list — marketing / spec / governance surfaces only. -# Historical progress notes (docs/*-progress.md), changelogs, and +# Historical progress notes (docs/*-progress.md), changelogs (and the entries a +# changelog's changes/ directory holds until a release renders them into it), and # external corpora are EXEMPT by design (they're snapshots, not the # message we're keeping in sync). Research and notes dirs also exempt. # Portable: read into an array without `mapfile` so macOS bash 3.2 works. files=() while IFS= read -r f; do case "$f" in - *-progress.md|*/docs/*-progress.md|*/CHANGELOG.md|*/conformance/*|*/research/*|*/notes/*|*/notes-*/*) + *-progress.md|*/docs/*-progress.md|*/CHANGELOG.md|*/changes/[0-9]*.md|*/conformance/*|*/research/*|*/notes/*|*/notes-*/*) continue ;; esac files+=("$f") diff --git a/scripts/check-tests-in-ci-selftest.sh b/scripts/check-tests-in-ci-selftest.sh index 8f63033d6..0dd35971c 100755 --- a/scripts/check-tests-in-ci-selftest.sh +++ b/scripts/check-tests-in-ci-selftest.sh @@ -61,5 +61,24 @@ expect fail "a listed test with no reason" "not .name reason." mk; printf '\n[[test]]\nname = "gone"\npath = "tests/gone.rs"\n' >> "$C/Cargo.toml" expect fail "a [[test]] entry for a file that does not exist" "which does not exist" +# A verifier step's own file names a test as a workflow does. +mk; printf '#[test]\nfn b() {}\n' > "$C/tests/beta.rs" +mkdir -p "$tmp/.github/wai-conformance/verifiers" +printf '# step: beta\n# order: 10\ncargo test --no-default-features --test beta\n' > "$tmp/.github/wai-conformance/verifiers/beta.sh" +expect pass "a test a verifier step's file names" +rm "$tmp/.github/wai-conformance/verifiers/beta.sh" +expect fail "and the same test once the file is gone" "beta.rs: no workflow step names it" + +# A tag a step takes names every test file that carries it, and only those. +mk; printf '// wai-conformance: int-kernels\n#[test]\nfn b() {}\n' > "$C/tests/beta.rs" +printf ' - run: python3 ../../scripts/wai-conformance-lists.py test-tagged int-kernels -- --lib\n' >> "$tmp/.github/workflows/w.yml" +expect pass "a test that carries a tag a step takes" +mk; printf '// wai-conformance: int-kernels-amd\n#[test]\nfn b() {}\n' > "$C/tests/beta.rs" +printf ' - run: python3 ../../scripts/wai-conformance-lists.py test-tagged int-kernels -- --lib\n' >> "$tmp/.github/workflows/w.yml" +expect fail "a test whose tag no step takes" "or takes a tag it carries" +mk; printf '// wai-conformance: int-kernels\n#[test]\nfn l() {}\n' > "$C/tests/local.rs" +printf ' - run: T=$(python3 ../../scripts/wai-conformance-lists.py tests int-kernels)\n' >> "$tmp/.github/workflows/w.yml" +expect fail "a test listed as not run that a step takes by its tag" "remove the line" + if [ $fails -ne 0 ]; then echo "check-tests-in-ci-selftest: $fails failure(s)"; exit 1; fi echo "check-tests-in-ci-selftest: the check passes a covered tree and fails each uncovered one." diff --git a/scripts/check-tests-in-ci.py b/scripts/check-tests-in-ci.py index b604c5385..b13ba55eb 100755 --- a/scripts/check-tests-in-ci.py +++ b/scripts/check-tests-in-ci.py @@ -7,10 +7,16 @@ crate is too large to run whole on every leg. So a test file nobody adds to a st runs, and nothing says so. And a file that is `#![cfg(feature = "...")]` compiles, in a step that lacks the feature, to a binary with no tests in it that reports `ok`. +A step may also take its tests from elsewhere: the steps of wai-conformance.yml's +independent verifiers are the files of `.github/wai-conformance/verifiers/`, and a step that +runs `scripts/wai-conformance-lists.py tests TAG` or `test-tagged TAG` runs every test file +whose line `// wai-conformance: TAG...` names TAG. Both count as naming the test. + For each crate listed in CRATES this requires that - 1. every `tests/*.rs` is named by `--test NAME` in a workflow, or is listed in the - crate's `tests/NOT-IN-CI.txt` with the reason no job runs it; + 1. every `tests/*.rs` is named by `--test NAME` in a workflow or a verifier step's file, + or carries a tag a workflow's step takes, or is listed in the crate's + `tests/NOT-IN-CI.txt` with the reason no job runs it; 2. every `tests/*.rs` whose crate-level `#![cfg(...)]` names features has a `[[test]]` entry in Cargo.toml whose `required-features` are exactly those features, so that naming it in a step without them stops cargo with an error; @@ -35,15 +41,42 @@ CRATES = ["wai/wai-rs"] NOT_IN_CI = "NOT-IN-CI.txt" +# The verifier steps wai-conformance.yml runs from their own files. +STEP_DIRS = [os.path.join(".github", "wai-conformance", "verifiers")] + + def workflow_test_names(root): - """Every NAME of a `--test NAME` in a workflow file.""" + """Every NAME of a `--test NAME` in a workflow file or a verifier step's file.""" names = set() wf = os.path.join(root, ".github", "workflows") + files = [os.path.join(wf, f) for f in sorted(os.listdir(wf)) if f.endswith((".yml", ".yaml"))] + for d in STEP_DIRS: + d = os.path.join(root, d) + if os.path.isdir(d): + files += [os.path.join(d, f) for f in sorted(os.listdir(d)) if f.endswith(".sh")] + for path in files: + with open(path, encoding="utf-8") as fh: + names.update(re.findall(r"--test[ =]([A-Za-z0-9_-]+)", fh.read())) + return names + + +def workflow_tags(root): + """Every tag a workflow step takes its tests by: `wai-conformance-lists.py tests TAG...` + or `test-tagged TAG...`.""" + tags = set() + wf = os.path.join(root, ".github", "workflows") for f in sorted(os.listdir(wf)): if f.endswith((".yml", ".yaml")): with open(os.path.join(wf, f), encoding="utf-8") as fh: - names.update(re.findall(r"--test[ =]([A-Za-z0-9_-]+)", fh.read())) - return names + for m in re.finditer(r"wai-conformance-lists\.py (?:tests|test-tagged)((?: [a-z0-9-]+)+)", fh.read()): + tags.update(m.group(1).split()) + return tags + + +def file_tags(path): + """The tags of a test file's `// wai-conformance:` lines.""" + with open(path, encoding="utf-8") as fh: + return {t for line in re.findall(r"^//\s*wai-conformance:\s*(.*?)\s*$", fh.read(), re.M) for t in line.split()} def crate_cfg_features(path): @@ -111,15 +144,17 @@ def not_in_ci(path): def check(root): findings = [] named = workflow_test_names(root) + tags = workflow_tags(root) for crate in CRATES: tdir = os.path.join(root, crate, "tests") files = sorted(f[:-3] for f in os.listdir(tdir) if f.endswith(".rs") and os.path.isfile(os.path.join(tdir, f))) declared = cargo_tests(os.path.join(root, crate, "Cargo.toml")) listed, bad = not_in_ci(os.path.join(tdir, NOT_IN_CI)) findings += bad + named = named | {t for t in files if file_tags(os.path.join(tdir, t + ".rs")) & tags} for t in files: if t not in named and t not in listed: - findings.append(f"{crate}/tests/{t}.rs: no workflow step names it (`--test {t}`), and {NOT_IN_CI} does not list it") + findings.append(f"{crate}/tests/{t}.rs: no workflow step names it (`--test {t}`) or takes a tag it carries, and {NOT_IN_CI} does not list it") feats = crate_cfg_features(os.path.join(tdir, t + ".rs")) if feats is None: findings.append(f"{crate}/tests/{t}.rs: its #![cfg(...)] is not a feature or an all(...) of features; this check cannot tell what it needs") diff --git a/scripts/pr-touches-paths-selftest.sh b/scripts/pr-touches-paths-selftest.sh index f0c143849..168b02ba5 100755 --- a/scripts/pr-touches-paths-selftest.sh +++ b/scripts/pr-touches-paths-selftest.sh @@ -82,4 +82,38 @@ Y check "a path listed after a comment and a blank line" true "$(run "$tmp/commented.yml" "$BASE" "$WAI")" check "and the list still says false for a diff outside it" false "$(run "$tmp/commented.yml" "$BASE" "$JCP")" +# --paths-from and --wide: the files of a directory bring their own paths, and the wide glob +# the forge filters on stands for them alone. A step's file (`# paths:` in its header) names +# one extension text; a change to it runs the workflow, and a change to an extension text no +# file names does not, though the forge's wide glob matches both. +mkdir -p "$repo/wai/extensions" "$repo/steps" +echo a > "$repo/wai/extensions/owned.md"; echo b > "$repo/wai/extensions/other.md" +printf '# step: one\n# order: 10\n# paths: wai/extensions/owned.md\n#\n# prose: not a header\necho ok\n' > "$repo/steps/one.sh" +printf '# step: two\n# paths: wai/extensions/second/**\necho ok\n# paths: wai/extensions/other.md\n' > "$repo/steps/two.sh" +git -C "$repo" add -A >/dev/null; git -C "$repo" commit -qm steps >/dev/null +S0=$(git -C "$repo" rev-parse HEAD) +echo changed > "$repo/wai/extensions/owned.md"; git -C "$repo" commit -qam owned >/dev/null; OWN=$(git -C "$repo" rev-parse HEAD) +echo changed > "$repo/wai/extensions/other.md"; git -C "$repo" commit -qam other >/dev/null; OTH=$(git -C "$repo" rev-parse HEAD) +mkdir -p "$repo/wai/extensions/second"; echo c > "$repo/wai/extensions/second/x.md" +git -C "$repo" add -A >/dev/null; git -C "$repo" commit -qm second >/dev/null; SEC=$(git -C "$repo" rev-parse HEAD) +cat > "$tmp/wide.yml" <<'Y' +on: + pull_request: + paths: + - 'wai/wai-pack/**' + - 'wai/extensions/**' +Y +wide() { ( cd "$repo" && python3 "$OLDPWD/$G" "$tmp/wide.yml" "$1" "$2" --paths-from steps --wide 'wai/extensions/**' ); } +check "a path a step's file names runs the workflow" true "$(wide "$S0" "$OWN")" +check "an extension text no file names does not, under the wide glob" false "$(wide "$OWN" "$OTH")" +check "a glob a file names matches below it" true "$(wide "$OTH" "$SEC")" +check "the workflow's own globs still count" true "$(wide "$BASE" "$WAI")" +check "without the flags the wide glob matches everything under it" true "$( cd "$repo" && python3 "$OLDPWD/$G" "$tmp/wide.yml" "$OWN" "$OTH" )" +check "a header line after the header is not read" false "$(printf '# step: two\necho ok\n# paths: wai/extensions/other.md\n' > "$repo/steps/two.sh"; wide "$OWN" "$OTH")" +# Every way of misreading them fails OPEN. +check "a --paths-from directory that does not exist" true "$( cd "$repo" && python3 "$OLDPWD/$G" "$tmp/wide.yml" "$OWN" "$OTH" --paths-from nowhere --wide 'wai/extensions/**' )" +check "a --wide glob the workflow does not list" true "$( cd "$repo" && python3 "$OLDPWD/$G" "$tmp/wide.yml" "$OWN" "$OTH" --paths-from steps --wide 'wai/nothing/**' )" +check "a --wide glob with no --paths-from" true "$( cd "$repo" && python3 "$OLDPWD/$G" "$tmp/wide.yml" "$OWN" "$OTH" --wide 'wai/extensions/**' )" +check "an argument the script does not read" true "$( cd "$repo" && python3 "$OLDPWD/$G" "$tmp/wide.yml" "$OWN" "$OTH" --what x )" + [ "$fail" = 0 ] && echo "pr-touches-paths self-test: all cases pass" || { echo "pr-touches-paths self-test FAILED"; exit 1; } diff --git a/scripts/pr-touches-paths.py b/scripts/pr-touches-paths.py index b27b24d5c..46c4576f6 100755 --- a/scripts/pr-touches-paths.py +++ b/scripts/pr-touches-paths.py @@ -13,7 +13,14 @@ This reads the path list out of the workflow file itself — one source of truth rather than a third copy of the list to drift from the two `on:` blocks — and compares it against `git diff --name-only BASE...HEAD`. - pr-touches-paths.py <workflow.yml> <base-sha> <head-sha> + pr-touches-paths.py <workflow.yml> <base-sha> <head-sha> [--paths-from DIR]... [--wide GLOB]... + +With --paths-from, the files of DIR add the globs their headers declare (leading lines +`# paths: <glob>...`, as the verifier steps of .github/wai-conformance/ do), and each --wide +glob of the workflow's list is dropped: it is the forge's filter for those files' paths, wide +because the forge cannot read them, and the gate replaces it with the paths themselves. So a +step that brings a path adds it in its own file, and the gate still answers for exactly the +paths named. Prints `true` or `false`. **Fails open**: anything it cannot determine prints `true`, because a wrong `false` silently skips a test and a wrong `true` only @@ -56,6 +63,45 @@ def paths_of(workflow: Path) -> list[str]: return globs +HEADER = re.compile(r"^# ([a-z][a-z-]*): ?(.*)$") + + +def declared_paths(directory: Path) -> "list[str] | None": + """The globs the files of `directory` declare: each `# paths:` line of a file's leading + `# key: value` lines. None when the directory holds no file, which fails open.""" + if not directory.is_dir(): + return None + files = sorted(p for p in directory.iterdir() if p.is_file() and not p.name.startswith(".")) + if not files: + return None + globs: list[str] = [] + for f in files: + for line in f.read_text(encoding="utf-8").splitlines(): + m = HEADER.match(line) + if not m: + break + if m.group(1) == "paths": + globs.extend(m.group(2).split()) + return globs + + +def gate_globs(workflow: Path, paths_from: "list[Path]", wide: "list[str]") -> "list[str] | None": + """The globs the gate answers for: the workflow's, each --wide one replaced by the paths + the --paths-from files declare. None when that cannot be read, which fails open.""" + globs = paths_of(workflow) + if not paths_from and not wide: + return globs + if not paths_from or not wide or any(w not in globs for w in wide): + return None # a wide glob with nothing to stand for, or one the workflow lacks + extra: list[str] = [] + for d in paths_from: + found = declared_paths(d) + if found is None: + return None + extra.extend(found) + return [g for g in globs if g not in wide] + extra + + def matches(path: str, glob: str) -> bool: """`glob` in the Actions sense: `**` spans separators, `*` does not.""" rx = [] @@ -77,14 +123,26 @@ def matches(path: str, glob: str) -> bool: def main() -> None: - if len(sys.argv) != 4: + args = sys.argv[1:] + paths_from: list[Path] = [] + wide: list[str] = [] + while len(args) > 3: + if args[-2] == "--paths-from": + paths_from.insert(0, Path(args[-1])) + elif args[-2] == "--wide": + wide.insert(0, args[-1]) + else: + print("true") # an argument this script does not read: run it + return + args = args[:-2] + if len(args) != 3: print("true") # cannot tell: run it return - workflow, base, head = Path(sys.argv[1]), sys.argv[2], sys.argv[3] + workflow, base, head = Path(args[0]), args[1], args[2] if not workflow.is_file() or not base or not head: print("true") return - globs = paths_of(workflow) + globs = gate_globs(workflow, paths_from, wide) if not globs: # No filter, or a shape this parser does not read: the workflow wanted # to run on everything, or we cannot say it did not. diff --git a/scripts/wai-conformance-lists-selftest.sh b/scripts/wai-conformance-lists-selftest.sh new file mode 100755 index 000000000..83b35f18c --- /dev/null +++ b/scripts/wai-conformance-lists-selftest.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# +# wai-conformance-lists-selftest.sh — the lists wai-conformance.yml derives can fail. +# +# The workflow's feature lists, test lists, steps, corpora and gate paths are read from files +# (scripts/wai-conformance-lists.py). A derivation that silently loses an entry drops coverage +# with every job still green, so this copies the files it reads into a scratch tree and +# requires: the tree as committed holds every entry of the hand lists it replaced; taking a +# step, a feature, a tag or a corpus away fails that; a new step's file brings its feature, +# test, binary and path into every list; each malformed file fails the check, saying why; and +# a step that fails stops the run with its status. Each check holds whatever else the tree +# holds, since this runs on every pull request, one that adds a step too: no check looks for an +# entry at a place in a sorted list, a step at a place in the run, or a line made by a file the +# check does not change itself. +set -uo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +repo=$(cd "$here/.." && pwd) +L="$here/wai-conformance-lists.py" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +T="$tmp/t" + +mk() { + rm -rf "$T" + mkdir -p "$T/scripts" "$T/wai/wai-rs/tests" "$T/.github" + cp -R "$repo/.github/workflows" "$repo/.github/wai-conformance" "$T/.github/" + cp "$repo/scripts/pr-touches-paths.py" "$T/scripts/" + cp "$repo/wai/wai-rs/Cargo.toml" "$T/wai/wai-rs/" + cp "$repo"/wai/wai-rs/tests/*.rs "$T/wai/wai-rs/tests/" +} +V="$T/.github/wai-conformance/verifiers" +ed() { perl -pi -e "$2" "$1"; } # ed <file> <perl substitution>, the same on GNU and BSD systems + +fails=0 +expect() { # expect <0|1|n> <what> <text the output must hold or ""> -- <lists.py args> + local want=$1 what=$2 needle=$3 out rc + shift 4 + out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$? + if [ "$want" = 0 ] && [ $rc -ne 0 ]; then echo "FAIL $what: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi + if [ "$want" != 0 ] && [ $rc -eq 0 ]; then echo "FAIL $what: it passed"; fails=$((fails+1)); return; fi + if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi + echo "ok $what" +} +# The lists are printed sorted, so what else the tree holds (a step a pull request adds, +# wherever its name sorts) moves an entry along a line: these find the entry among the line's +# words instead of at a fixed place in it. +named() { # named <output> <lead> <entry>: a line `FAIL <lead> ...` names <entry> among its words + local lead="FAIL $2 " l w words + while IFS= read -r l; do + [[ $l == "$lead"* ]] || continue + read -ra words <<<"${l#"$lead"}" + for w in "${words[@]}"; do [ "$w" = "$3" ] && return 0; done + done <<<"$1" + return 1 +} +expect_named() { # expect_named <what> <lead> <entry> [<lead> <entry>]... -- <lists.py args>: it fails, naming each + local what=$1 out rc miss="" pairs=() i + shift + while [ "$1" != -- ]; do pairs+=("$1" "$2"); shift 2; done + shift + out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$? + if [ $rc -eq 0 ]; then echo "FAIL $what: it passed"; fails=$((fails+1)); return; fi + for ((i = 0; i < ${#pairs[@]}; i += 2)); do + named "$out" "${pairs[i]}" "${pairs[i+1]}" || miss="$miss; no line \`${pairs[i]} ...\` names ${pairs[i+1]}" + done + if [ -n "$miss" ]; then echo "FAIL $what${miss/;/:}"; grep '^FAIL [A-Za-z_]*: ' <<<"$out" | head -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi + echo "ok $what" +} + +# The tree as committed. +mk +expect 0 "the committed files hold every entry of the hand lists" "steps of the hand list run" -- selftest +expect 0 "and the check finds nothing" "well formed" -- check +a=$(python3 -I "$L" --root "$T" dry-run); b=$(python3 -I "$L" --root "$T" dry-run) +if [ "$a" = "$b" ] && [ -n "$a" ]; then echo "ok dry-run prints the same twice"; else echo "FAIL dry-run is not stable"; fails=$((fails+1)); fi +python3 -I "$L" --root "$T" prebuild > "$tmp/prebuild.sh" && bash -n "$tmp/prebuild.sh" \ + && echo "ok the build step's script is shell" || { echo "FAIL the build step's script does not parse"; fails=$((fails+1)); } + +# Each way coverage can leave a list fails. +mk; rm "$V/live-events.sh" +expect_named "a verifier step's file taken away" "WAI_FEATURES: lost" live_events "union_tests: lost" live_events -- selftest +expect 1 "and its step named" "step \`live events · wai-rs and live_events_verify.py\` no longer runs" -- selftest +mk; ed "$V/live-rooms.sh" 's/^# features: live_room$/# features: live_compose/' +expect_named "a feature taken out of a step's header" "WAI_FEATURES: lost" live_room -- selftest +mk; perl -ni -e 'print unless $. == 1' "$T/wai/wai-rs/tests/int_kernels_vectors.rs" +expect_named "a tag taken off a test file" "int_kernels_debug_tests: lost" int_kernels_vectors -- selftest +mk; rm "$T/.github/wai-conformance/corpora/worlds.txt" +expect 1 "a corpus file taken away" "corpora: lost" -- selftest +mk; ed "$T/.github/workflows/wai-conformance.yml" 's/--paths-from .github\/wai-conformance\/verifiers //' +expect 1 "the gate given its wide glob without the files' paths" "it would run on every change" -- selftest + +# A new step brings its own entries into every list, and edits no other file. +mk +# What `selftest --exact` says of the steps and of the order, for the tree as committed. +exact_steps() { python3 -I "$L" --root "$T" selftest --exact 2>&1 | grep -E '^FAIL (step |the steps |the corpora )' | LC_ALL=C sort; } +s0=$(exact_steps) +python3 - "$T/wai/wai-rs/Cargo.toml" <<'P' +import sys +p = sys.argv[1] +t = open(p).read().replace("[features]\n", "[features]\nfresh_step = []\n", 1) +open(p, "w").write(t) +P +printf '#[test]\nfn f() {}\n' > "$T/wai/wai-rs/tests/fresh_step.rs" +# Its order is below every step's in the tree, so it runs first whatever orders the tree holds. +lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$V"/*.sh | sort -n | head -1) +cat > "$V/fresh-step.sh" <<S +# step: fresh step · wai-rs and fresh_verify.py +# order: $((lo - 1)) +# features: fresh_step +# bins: wai_claim_vectors +# paths: wai/extensions/fresh-step.md +# +# A step the self-test adds. +set -euo pipefail +echo "the step ran with \$WAI_FEATURES" +(cd wai-rs && cargo test --no-default-features --features "\$WAI_FEATURES" --test fresh_step) +S +expect 0 "a new step's file: the lists still hold the hand lists" "steps of the hand list run" -- selftest +expect_named "and are more than them" "WAI_FEATURES: more than the hand list:" fresh_step \ + "union_tests: more than the hand list:" fresh_step "alone: more than the hand list:" fresh_step \ + "paths: more than the hand list:" wai/extensions/fresh-step.md -- selftest --exact +expect 0 "its feature joins WAI_FEATURES, first by its order" "WAI_FEATURES=fresh_step," -- env +expect 0 "its test joins the build" "--test fresh_step" -- prebuild +expect 0 "its feature list is checked alone" "fresh_step" -- alone +expect 0 "its path joins the gate" "wai/extensions/fresh-step.md" -- paths +expect 0 "its step runs first, by its order" " - step: fresh step" -- dry-run +first=$(python3 -I "$L" --root "$T" dry-run | grep -m1 '^ - step:') +[ "$first" = " - step: fresh step · wai-rs and fresh_verify.py (working-directory: wai)" ] \ + && echo "ok and before every other" || { echo "FAIL the new step is not first: $first"; fails=$((fails+1)); } +# --exact compares each step by its name, not its place: the step planted first is one +# addition, and every step of the hand list keeps its body and its order. +s1=$(exact_steps) +more=$(LC_ALL=C comm -13 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1")) +less=$(LC_ALL=C comm -23 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1")) +[ "$more" = 'FAIL step `fresh step · wai-rs and fresh_verify.py`: more than the hand list' ] && [ -z "$less" ] \ + && echo "ok --exact names the step planted first as one addition, and no step of the hand list as changed or moved" \ + || { echo "FAIL --exact, a step planted first:"; printf '%s\n' "$more" | head -5 | sed 's/^/ + /'; printf '%s\n' "$less" | head -5 | sed 's/^/ - /'; fails=$((fails+1)); } +# And a corpus planted first is one addition, not a change of the corpora's order. +C="$T/.github/wai-conformance/corpora" +lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$C"/*.txt | sort -n | head -1) +printf '# corpus: fresh\n# order: %s\n# bin: wai_world_vectors\n# dir: ../fresh-conformance\n' "$((lo - 1))" > "$C/fresh.txt" +out=$(python3 -I "$L" --root "$T" selftest --exact 2>&1) +if grep -qE '^FAIL corpora: more than the hand list: (.* )?fresh wai_world_vectors \.\./fresh-conformance( |$)' <<<"$out" && [ "$(exact_steps)" = "$s1" ]; then + echo "ok --exact names a corpus planted first as an addition, and not as a change of order" +else + echo "FAIL --exact, a corpus planted first:"; grep '^FAIL' <<<"$out" | head -5 | sed 's/^/ /'; fails=$((fails+1)) +fi + +# The dry-run prints the command lines of the derived steps outside the verifiers' job, so a +# change to one of them shows in a diff of two dry-runs, and it refuses a step it cannot read. +mk +d0=$(python3 -I "$L" --root "$T" dry-run) +expect 0 "the dry-run prints the release kernel step's command" "$ cargo test --lib --no-default-features --release --features " -- dry-run +expect 0 "and int-kernels-amd's, both profiles" "$ nice -n 10 cargo test --lib --locked --no-default-features --release --features " -- dry-run +# Each of whatever corpora and feature lists the tree holds, by its own line. +tails=$(grep -o -- '--bin [^ ]* -- verify [^ ]*$' <<<"$d0") +cs=$(python3 -I "$L" --root "$T" corpora) +miss=$(while read -r c b dir; do [ -n "$c" ] || continue; grep -qxF -- "--bin $b -- verify $dir" <<<"$tails" || printf ' %s' "$c"; done <<<"$cs") +[ -n "$cs" ] && [ -z "$miss" ] && echo "ok and each corpus's" \ + || { echo "FAIL the dry-run prints no command for the corpora:${miss:- (none listed)}"; fails=$((fails+1)); } +checks=$(grep -o -- '\$ cargo check --no-default-features --tests --features [^ ]*$' <<<"$d0") +as=$(python3 -I "$L" --root "$T" alone) +miss=$(while read -r a; do + [ -n "$a" ] || continue + n=$(tr ',' '\n' <<<"$a" | LC_ALL=C sort | paste -sd, -) + grep -qxF -- "\$ cargo check --no-default-features --tests --features $n" <<<"$checks" || printf ' %s' "$a" +done <<<"$as") +[ -n "$as" ] && [ -z "$miss" ] && echo "ok and each list features-alone checks" \ + || { echo "FAIL the dry-run prints no features-alone check for the lists:${miss:- (none listed)}"; fails=$((fails+1)); } +ed "$T/.github/workflows/wai-conformance.yml" 's/(int-kernels int-kernels-release --extra-features i32_forms -- )--release /$1/' +d1=$(python3 -I "$L" --root "$T" dry-run) +dd=$(diff <(printf '%s\n' "$d0") <(printf '%s\n' "$d1")) +n=$(grep -c '^[<>]' <<<"$dd") +[ "$n" = 2 ] && grep -q '^< .*--release --features' <<<"$dd" \ + && echo "ok a flag taken off a kernel step is one changed line of the dry-run" \ + || { echo "FAIL taking --release off the release step changed $n line(s) of the dry-run"; fails=$((fails+1)); } +mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+for f in \$lists; do)$/$1\n cargo check \$NOT_SET/' +expect 1 "a command reading a value the dry-run cannot know fails it" "\`\$NOT_SET\` has no value the dry-run knows" -- dry-run +mk; ed "$T/.github/workflows/wai-conformance.yml" 's/(run: python3 \.\.\/\.\.\/scripts\/wai-conformance-lists\.py corpora --verify.*)$/$1\n if: matrix.os != '"'"'windows'"'"'/' +expect 0 "a step one shell family skips prints each family's commands" "every cargo-class corpus (the Windows leg; working-directory: -)" -- dry-run +mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+- name: )wai-rs, each feature list alone$/$1wai-rs, every feature list alone/' +expect 1 "a derived step the dry-run cannot find fails it" "has no step \`wai-rs, each feature list alone...\`" -- dry-run + +# Each malformed file fails the check, saying why. +mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: 100\n# feature: live_events/' +expect 1 "a header key no step file has" "\`feature\` is not a key" -- check +mk; perl -ni -e 'print unless /^# step: /' "$V/live-events.sh" +expect 1 "a step file with no step name" "no \`step:\` line" -- check +mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: soon/' +expect 1 "an order that is no integer" "is not an integer" -- check +mk; ed "$V/live-events.sh" 's#^\# paths: wai/extensions/live-events.md#\# paths: docs/live-events.md#' +expect 1 "a path the forge's filter does not cover" "so the forge would never start the workflow for it" -- check +mk; ed "$V/live-events.sh" 's/^# features: live_events$/# features: live_eventz/' +expect 1 "a feature Cargo.toml does not have" "\`live_eventz\` is not in" -- check +mk; ed "$V/live-events.sh" 's/--test live_events\)/--test live_eventz)/' +expect 1 "a test the crate does not have" "--test live_eventz: wai/wai-rs has no such test" -- check +mk; ed "$V/http-differential.sh" 's#wai-bins/union/wai_http_vectors#wai-bins/union/wai_http_vectorz#' +expect 1 "a binary the crate does not have" "\`wai_http_vectorz\` is not a [[bin]]" -- check +mk; ed "$T/wai/wai-rs/tests/int_kernels_vectors.rs" 's{^// wai-conformance: int-kernels }{// wai-conformance: int-kernel }' +expect 1 "a tag no step takes" "is no tag a step takes" -- check +mk; cp "$V/live-events.sh" "$V/live-events-again.sh" +expect 1 "two files with one step name" "is also" -- check +mk; printf '# step: empty\n# order: 1\n#\n# nothing to run\n' > "$V/empty.sh" +expect 1 "a step file with no body" "no body" -- check +mk; printf '# corpus: x\n# order: 1\n# bin: wai_world_vectors\n' > "$T/.github/wai-conformance/corpora/x.txt" +expect 1 "a corpus file with no directory" "no \`dir:\` line" -- check + +# The commands refuse what they cannot answer, rather than answering with nothing. +mk +expect 1 "a tag no step takes, asked for" "a tag is one of" -- tests int-kernel +for f in "$T"/wai/wai-rs/tests/*.rs; do perl -ni -e 'print unless m{^// wai-conformance:}' "$f"; done +expect 1 "tests for a tag no file carries" "the step would run no test" -- tests int-kernels +expect 1 "and test-tagged with it" "the step would run no test" -- test-tagged int-kernels -- --lib + +# A step that fails stops the run with its status, and names its file. +mk +printf '# step: passes\n# order: 1\nset -euo pipefail\necho "features: $WAI_FEATURES"\n' > "$V/aa-passes.sh" +printf '# step: fails\n# order: 2\nset -euo pipefail\nexit 3\n' > "$V/ab-fails.sh" +# The features it is given are the whole derived list, as `env` prints it for the tree. +wf=$(python3 -I "$L" --root "$T" env | sed -n 's/^WAI_FEATURES=//p') +out=$(python3 -I "$L" --root "$T" run --only aa-passes 2>&1); rc=$? +if [ $rc -eq 0 ] && [ -n "$wf" ] && grep -qxF -- "features: $wf" <<<"$out"; then echo "ok a step that passes runs, with the derived features" +else echo "FAIL a passing step: exit $rc, or not given WAI_FEATURES=$wf"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); fi +out=$(python3 -I "$L" --root "$T" run --only aa-passes --only ab-fails 2>&1); rc=$? +if [ $rc -eq 3 ] && grep -qF "::error::fails failed with exit status 3 (.github/wai-conformance/verifiers/ab-fails.sh)" <<<"$out"; then + echo "ok a step that fails stops the run with its status, naming its file" +else + echo "FAIL a failing step: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)) +fi +printf '# step: pipe\n# order: 3\nfalse | true\n' > "$V/ac-pipe.sh" +expect 1 "a step whose pipe fails fails, as under a workflow's bash" "" -- run --only ac-pipe +expect 1 "a step --only names that no file holds" "names a step no file holds" -- run --only no-such-step + +if [ $fails -ne 0 ]; then echo "wai-conformance-lists-selftest: $fails failure(s)"; exit 1; fi +echo "wai-conformance-lists-selftest: the derived lists hold the hand lists, and fail each way they can lose an entry." diff --git a/scripts/wai-conformance-lists.py b/scripts/wai-conformance-lists.py new file mode 100755 index 000000000..2f36e769c --- /dev/null +++ b/scripts/wai-conformance-lists.py @@ -0,0 +1,1021 @@ +#!/usr/bin/env python3 +"""wai-conformance-lists.py: the lists wai-conformance.yml runs, derived from the files that +own them, so a change adds a file instead of editing the workflow's shared lists. + +Each list below used to be kept by hand in the workflow, and every new module of WAI appended +to it, so any two branches that added one conflicted. Each is now read at run time from its +source: + + .github/wai-conformance/verifiers/*.sh one file a step of the independent verifiers: its + shell body, and a header of `# key: value` lines that + gives its features, binaries, paths and order. From + them: WAI_FEATURES, WAI_PQ_FEATURES and + WAI_RELEASE_FEATURES, the binaries and tests the job + builds first, the steps it runs, the feature lists the + features-alone job checks, and the paths the workflow's + gate runs on. + .github/wai-conformance/corpora/*.txt one file a cargo-class corpus: its tool and directory. + The cargo-classes corpus loop runs them, with the + features their tools require (Cargo.toml) and the + step's own. + wai/wai-rs/tests/*.rs a test file whose line `// wai-conformance: <tag>...` + names a tag is run by the steps that take that tag: + int-kernels (both profiles), int-kernels-release (the + release profile only), int-kernels-amd (the AVX-512 + host). Their features are the tests' required-features + and the step's own. + +A verifier file's header (the leading `# key: value` lines; the first other line ends it): + + step its name in the log (required) + order an integer: steps run in ascending order, a tie by file name (required) + features feature lists, space-separated, each comma-joined: each compiles alone in + the features-alone job, and every feature in them joins the build's list + alone the lists features-alone checks instead of `features`; `-` for none + build union (the default: WAI_FEATURES) or pq (WAI_PQ_FEATURES) + release-features features of the optimised build (WAI_RELEASE_FEATURES) + bins, pq-bins binaries the job builds first and copies to $RUNNER_TEMP/wai-bins/union or + /pq, beside every `wai-bins/union/NAME` and `wai-bins/pq/NAME` the body names + paths globs whose change runs the workflow (each under one of its `paths:` globs) + working-directory where the body runs, from the repository root (default wai) + +The tests the job builds first are the `--test NAME` of the body's lines that build with +"$WAI_FEATURES", and with "$WAI_RELEASE_FEATURES" for the optimised build. + +Commands (the workflow runs the first group): + + env WAI_FEATURES=..., WAI_PQ_FEATURES=..., WAI_RELEASE_FEATURES=... + prebuild the shell script that builds the binaries and tests first + run [--only SLUG]... every verifier step in order, each in a log group; the first that + fails stops the run with its exit status + alone the feature lists features-alone checks, one a line + paths the paths the verifier files bring to the gate, one a line + tests TAG... `--test NAME` for every test file that carries one of the tags + features TAG... [--extra-features A,B] + the tests' required-features with A,B, comma-joined + test-tagged TAG... [--extra-features A,B] -- ARGS... + cargo test ARGS --features <features> <tests>, printed, then run + corpora [--verify] [--extra-features A,B] + the corpora, `name bin dir` a line; with --verify, run each tool's + verify over its corpus (cargo run), each in a log group + dry-run [--ref REF] every list and every step the workflow would run, in a stable form; + of the tree at REF instead with --ref, read from its files or, before + they existed, from the lists its workflow kept by hand. It ends with + the cargo command lines of the steps outside the verifiers' job that + take derived lists (the integer kernels' two steps, the corpus step, + int-kernels-amd, the features-alone loop), read from the workflow's own + step text and evaluated, each in one form (flags and features + sorted, each target once), and per shell family when the unix legs + and the Windows leg differ. A reviewer diffs two dry-runs: the same + lines are the same builds and tests. + baseline --ref REF the lists REF's workflow kept by hand, as JSON + check every file above is well formed and names what exists + selftest [--exact] check, then every list derived here holds every entry of + .github/wai-conformance/baseline.json (the hand lists of the commit + that moved them out), and every step it names still runs; --exact + requires the lists equal, and each step the same body and directory + as the hand list's step of its name, in the same order (a step or a + corpus the tree adds is named as an addition) + +Standard library only; runs on the Python 3.9 the runner images carry. --root DIR reads +another tree. +""" +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +VERIFIERS = ".github/wai-conformance/verifiers" +CORPORA = ".github/wai-conformance/corpora" +BASELINE = ".github/wai-conformance/baseline.json" +WORKFLOW = ".github/workflows/wai-conformance.yml" +CRATE = "wai/wai-rs" +TAGS = ("int-kernels", "int-kernels-release", "int-kernels-amd") + +VERIFIER_KEYS = { + "step": "one", "order": "one", "features": "many", "alone": "many", "build": "one", + "release-features": "many", "bins": "many", "pq-bins": "many", "paths": "many", + "working-directory": "one", +} +CORPUS_KEYS = {"corpus": "one", "order": "one", "bin": "one", "dir": "one"} +HEADER = re.compile(r"^# ([a-z][a-z-]*): ?(.*)$") +MARKER = re.compile(r"^//\s*wai-conformance:\s*(.*?)\s*$", re.M) + + +class Bad(Exception): + pass + + +def uniq(xs): + out, seen = [], set() + for x in xs: + if x not in seen: + seen.add(x) + out.append(x) + return out + + +def gate_module(root): + """scripts/pr-touches-paths.py, imported: the gate's own reading of paths and headers.""" + path = os.path.join(root, "scripts", "pr-touches-paths.py") + if not os.path.isfile(path): + path = os.path.join(HERE, "pr-touches-paths.py") + spec = importlib.util.spec_from_file_location("pr_touches_paths", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# ---------------------------------------------------------------------------- the files + + +def read_header(path, keys, label): + """(header, body, errors) of a file whose leading `# key: value` lines are its header; + `label` names it in the errors. The body is what follows the leading comment block + (header and prose).""" + with open(path, encoding="utf-8") as fh: + lines = fh.read().split("\n") + head, errs, i = {}, [], 0 + while i < len(lines): + m = HEADER.match(lines[i]) + if not m: + break + k, v = m.group(1), m.group(2).strip() + if k not in keys: + errs.append(f"{label}: line {i + 1}: `{k}` is not a key of this file ({', '.join(sorted(keys))})") + elif keys[k] == "one": + if k in head: + errs.append(f"{label}: line {i + 1}: `{k}` is given twice") + head[k] = v + else: + head.setdefault(k, []).extend(v.split()) + i += 1 + while i < len(lines) and lines[i].startswith("#"): + i += 1 + body = "\n".join(lines[i:]).rstrip("\n") + return head, body, errs + + +def joined(body): + """The body's lines with `\\` continuations joined.""" + return re.sub(r"\\\n\s*", " ", body).split("\n") + + +def verifiers(root): + d = os.path.join(root, VERIFIERS) + if not os.path.isdir(d): + raise Bad(f"{VERIFIERS}: no such directory") + mods, errs = [], [] + for f in sorted(os.listdir(d)): + if not f.endswith(".sh"): + continue + rel = f"{VERIFIERS}/{f}" + h, body, e = read_header(os.path.join(d, f), VERIFIER_KEYS, rel) + errs += e + for k in ("step", "order"): + if k not in h: + errs.append(f"{rel}: no `{k}:` line in its header") + try: + order = int(h.get("order", "0")) + except ValueError: + errs.append(f"{rel}: order `{h.get('order')}` is not an integer") + order = 0 + build = h.get("build", "union") + if build not in ("union", "pq"): + errs.append(f"{rel}: build `{build}` is neither union nor pq") + alone = h.get("alone") + if alone == ["-"]: + alone = [] + elif alone is not None and "-" in alone: + errs.append(f"{rel}: alone `-` stands alone") + if not body.strip(): + errs.append(f"{rel}: no body: the step would run nothing") + mods.append({ + "slug": f[:-3], "file": rel, "step": h.get("step", ""), "order": order, + "features": h.get("features", []), "alone": alone, "build": build, + "release": h.get("release-features", []), "bins": h.get("bins", []), + "pq_bins": h.get("pq-bins", []), "paths": h.get("paths", []), + "wd": h.get("working-directory", "wai"), "body": body, + }) + mods.sort(key=lambda m: (m["order"], m["slug"])) + seen = {} + for m in mods: + if m["step"] and m["step"] in seen: + errs.append(f"{m['file']}: step `{m['step']}` is also {seen[m['step']]}'s") + seen.setdefault(m["step"], m["file"]) + return mods, errs + + +def corpora(root): + d = os.path.join(root, CORPORA) + if not os.path.isdir(d): + raise Bad(f"{CORPORA}: no such directory") + out, errs = [], [] + for f in sorted(os.listdir(d)): + if not f.endswith(".txt"): + continue + rel = f"{CORPORA}/{f}" + h, body, e = read_header(os.path.join(d, f), CORPUS_KEYS, rel) + errs += e + for k in CORPUS_KEYS: + if k not in h: + errs.append(f"{rel}: no `{k}:` line") + if body.strip(): + errs.append(f"{rel}: holds lines after its header; a corpus file is its header alone") + try: + order = int(h.get("order", "0")) + except ValueError: + errs.append(f"{rel}: order `{h.get('order')}` is not an integer") + order = 0 + out.append({"slug": f[:-4], "file": rel, "corpus": h.get("corpus", ""), "order": order, + "bin": h.get("bin", ""), "dir": h.get("dir", "")}) + out.sort(key=lambda c: (c["order"], c["slug"])) + return out, errs + + +def cargo_targets(root): + """{(kind, name): required-features or []} of the crate's [[bin]] and [[test]] entries, + and the feature names of its [features] table.""" + with open(os.path.join(root, CRATE, "Cargo.toml"), encoding="utf-8") as fh: + text = fh.read() + targets = {} + for block in re.split(r"^\[\[", text, flags=re.M)[1:]: + kind = block.split("]]", 1)[0] + body = re.split(r"^\[", block[len(kind) + 2:], flags=re.M)[0] + name = re.search(r'^name\s*=\s*"([^"]+)"', body, re.M) + if not name: + continue + rf = re.search(r"^required-features\s*=\s*\[(.*?)\]", body, re.M | re.S) + targets[(kind, name.group(1))] = sorted(re.findall(r'"([^"]+)"', rf.group(1))) if rf else [] + sec = re.search(r"^\[features\]\n(.*?)(?=^\[)", text, re.M | re.S) + feats = set(re.findall(r"^([A-Za-z0-9_-]+)\s*=", sec.group(1), re.M)) if sec else set() + return targets, feats + + +def markers(root): + """{test name: [tags]} of the crate's test files that carry a `wai-conformance:` line.""" + tdir = os.path.join(root, CRATE, "tests") + out = {} + for f in sorted(os.listdir(tdir)): + if not f.endswith(".rs"): + continue + with open(os.path.join(tdir, f), encoding="utf-8") as fh: + found = MARKER.findall(fh.read()) + if found: + out[f[:-3]] = uniq(t for line in found for t in line.split()) + return out + + +# ------------------------------------------------------------------------- derivations + + +def derive(mods): + union = [m for m in mods if m["build"] == "union"] + pq = [m for m in mods if m["build"] == "pq"] + feats = lambda ms: uniq(f for m in ms for lst in m["features"] for f in lst.split(",")) + tests_with = lambda var: uniq( + t for m in mods for line in joined(m["body"]) if f'"${var}"' in line for t in re.findall(r"--test[ =]([A-Za-z0-9_-]+)", line)) + bins_of = lambda key, build: uniq( + b for m in mods for b in m[key] + re.findall(rf"wai-bins/{build}/([A-Za-z0-9_-]+)", m["body"])) + return { + "WAI_FEATURES": feats(union), + "WAI_PQ_FEATURES": feats(pq), + "WAI_RELEASE_FEATURES": uniq(f for m in mods for f in m["release"]), + "union_bins": bins_of("bins", "union"), + "pq_bins": bins_of("pq_bins", "pq"), + "union_tests": tests_with("WAI_FEATURES"), + "release_tests": tests_with("WAI_RELEASE_FEATURES"), + "alone": uniq(lst for m in mods for lst in (m["features"] if m["alone"] is None else m["alone"])), + "paths": uniq(p for m in mods for p in m["paths"]), + } + + +def tagged(root, tags): + return sorted(t for t, ts in markers(root).items() if set(ts) & set(tags)) + + +def tagged_features(root, tags, extra): + targets, _ = cargo_targets(root) + out = set(extra) + for t in tagged(root, tags): + out |= set(targets.get(("test", t)) or []) + return sorted(out) + + +def corpora_features(root, cs, extra): + targets, _ = cargo_targets(root) + out = set(extra) + for c in cs: + out |= set(targets.get(("bin", c["bin"])) or []) + return sorted(out) + + +def prebuild(d): + """The build step's script: the binaries the steps run, copied per build, then the test + builds every step finds fresh. The order matters (see the step's comment).""" + s = ["set -euo pipefail", 'mkdir -p "$RUNNER_TEMP/wai-bins/union" "$RUNNER_TEMP/wai-bins/pq"'] + def build(var, bins, dest): + if not bins: + return + s.append(f'cargo build --no-default-features --features "${var}" \\') + s.append(" " + " ".join(f"--bin {b}" for b in bins)) + s.append(f"for b in {' '.join(bins)}; do") + s.append(f' cp "${{CARGO_TARGET_DIR:-target}}/debug/$b" "$RUNNER_TEMP/wai-bins/{dest}/"') + s.append("done") + build("WAI_FEATURES", d["union_bins"], "union") + if d["pq_bins"]: + s.append("# Copied before this build replaces target/debug's copies of the ones it shares.") + build("WAI_PQ_FEATURES", d["pq_bins"], "pq") + s.append('cargo test --no-run --no-default-features --features "$WAI_FEATURES" --lib \\') + s.append(" " + " ".join(f"--test {t}" for t in d["union_tests"])) + if d["release_tests"]: + s.append('cargo test --release --no-run --no-default-features --features "$WAI_RELEASE_FEATURES" \\') + s.append(" " + " ".join(f"--test {t}" for t in d["release_tests"])) + return "\n".join(s) + "\n" + + +def workflow_extras(text): + """The features the workflow's steps add to their derived lists: {command: [features]}, + the command being what precedes `--extra-features` on the step's line.""" + out = {} + for m in re.finditer(r"wai-conformance-lists\.py ((?:[a-z-]+ )+)--extra-features ([A-Za-z0-9_,-]+)", text): + out.setdefault(m.group(1).strip(), m.group(2).split(",")) + return out + + +def wide_globs(text): + """The globs the workflow's gate replaces with the verifier files' paths (--wide).""" + return re.findall(r"--wide '([^']+)'", text) + + +def gate_paths(root): + """What the gate runs on, as the gate reads it (scripts/pr-touches-paths.py with the + workflow's --paths-from and --wide): the workflow's paths but the wide ones, and the + verifier files' paths.""" + gate = gate_module(root) + from pathlib import Path + wf = os.path.join(root, WORKFLOW) + with open(wf, encoding="utf-8") as fh: + text = fh.read() + dirs = [Path(os.path.join(root, d)) for d in re.findall(r"--paths-from (\S+)", text)] + globs = gate.gate_globs(Path(wf), dirs, wide_globs(text)) + if globs is None: + raise Bad(f"the gate cannot read {WORKFLOW}'s paths with its --paths-from and --wide: it would run on every change") + return uniq(globs) + + +def from_tree(root): + """Every list, as this tree's files give them.""" + mods, errs = verifiers(root) + cs, e2 = corpora(root) + if errs + e2: + raise Bad("\n".join(errs + e2)) + with open(os.path.join(root, WORKFLOW), encoding="utf-8") as fh: + text = fh.read() + x = workflow_extras(text) + d = derive(mods) + dbg = tagged(root, ["int-kernels"]) + rel = tagged(root, ["int-kernels", "int-kernels-release"]) + return { + "paths": gate_paths(root), + "WAI_FEATURES": d["WAI_FEATURES"], "WAI_PQ_FEATURES": d["WAI_PQ_FEATURES"], + "WAI_RELEASE_FEATURES": d["WAI_RELEASE_FEATURES"], + "union_bins": d["union_bins"], "pq_bins": d["pq_bins"], + "union_tests": d["union_tests"], "release_tests": d["release_tests"], + "alone": d["alone"], + "steps": [{"step": m["step"], "wd": m["wd"], "body": m["body"]} for m in mods], + "int_kernels_debug_features": tagged_features(root, ["int-kernels"], x.get("test-tagged int-kernels", [])), + "int_kernels_debug_tests": dbg, + "int_kernels_release_features": tagged_features(root, ["int-kernels", "int-kernels-release"], x.get("test-tagged int-kernels int-kernels-release", [])), + "int_kernels_release_tests": rel, + "int_kernels_amd_features": tagged_features(root, ["int-kernels"], x.get("features int-kernels", [])), + "int_kernels_amd_tests": tagged(root, ["int-kernels-amd"]), + "corpora_features": corpora_features(root, cs, x.get("corpora --verify", [])), + "corpora": [f"{c['corpus']} {c['bin']} {c['dir']}" for c in cs], + } + + +# ------------------------------------------------- the lists a workflow kept by hand + + +def job_steps(text, job): + """[{name, wd, body, if, shell}] of a job's steps, read from the workflow text.""" + lines = text.split("\n") + start = next((i for i, l in enumerate(lines) if l == f" {job}:"), None) + if start is None: + raise Bad(f"{WORKFLOW} has no job `{job}`") + end = next((i for i in range(start + 1, len(lines)) if re.match(r"^ [a-z0-9_-]+:\s*$", lines[i])), len(lines)) + out, i = [], start + while i < end: + m = re.match(r"^ - name: (.*)$", lines[i]) + i += 1 + if not m: + continue + s = {"name": m.group(1).strip(), "wd": None, "body": None, "if": None, "shell": None} + while i < end and lines[i].startswith(" "): + k = re.match(r"^ ([a-z-]+): ?(.*)$", lines[i]) + if k and k.group(1) == "run" and k.group(2) == "|": + b = [] + i += 1 + while i < end and (lines[i].startswith(" ") or not lines[i].strip()): + b.append(lines[i][10:]) + i += 1 + s["body"] = "\n".join(b).rstrip("\n") + continue + if k and k.group(1) == "run": + s["body"] = k.group(2) + elif k and k.group(1) == "working-directory": + s["wd"] = k.group(2) + elif k and k.group(1) in ("if", "shell"): + s[k.group(1)] = k.group(2).strip() + i += 1 + out.append(s) + return out + + +def steps_of_job(text, job): + """[(name, working-directory, run body)] of a job's steps, read from the workflow text.""" + return [(s["name"], s["wd"], s["body"]) for s in job_steps(text, job)] + + +def from_hand_lists(root): + """Every list, as a workflow that kept them by hand states them (before this change).""" + gate = gate_module(root) + from pathlib import Path + wf = os.path.join(root, WORKFLOW) + with open(wf, encoding="utf-8") as fh: + text = fh.read() + env = lambda k: re.search(rf"^ {k}: (.*)$", text, re.M).group(1).split(",") + iv = steps_of_job(text, "independent-verifiers") + b = next(n for n, s in enumerate(iv) if s[0].startswith("Build wai-rs once")) + build = joined(iv[b][2]) + pick = lambda pat: next(l for l in build if re.search(pat, l)) + union_build = pick(r'cargo build .*"\$WAI_FEATURES"') + pq_build = pick(r'cargo build .*"\$WAI_PQ_FEATURES"') + union_test = pick(r'cargo test --no-run .*"\$WAI_FEATURES"') + rel_test = pick(r'cargo test --release --no-run') + fa = steps_of_job(text, "features-alone") + loop = re.search(r"for f in (.*?); do", " ".join(joined(fa[[s[0] for s in fa].index("wai-rs, each feature list alone")][2]))) + cc = steps_of_job(text, "cargo-classes") + ik = lambda name: next(s[2] for s in cc if s[0] == name) + dbg = ik("Fast integer kernels ≡ reference kernels (overflow checks on)") + rel = ik("Fast integer kernels ≡ reference kernels (release, overflow checks off)") + corp = ik("Every cargo-class corpus reproduces its frozen hashes (unix)") + amd = steps_of_job(text, "int-kernels-amd")[-1][2] + tests = lambda l: re.findall(r"--test ([A-Za-z0-9_-]+)", l) + feats = lambda l: re.search(r"--features (\S+)", l).group(1).split(",") + return { + "paths": uniq(gate.paths_of(Path(wf))), + "WAI_FEATURES": env("WAI_FEATURES"), "WAI_PQ_FEATURES": env("WAI_PQ_FEATURES"), + "WAI_RELEASE_FEATURES": env("WAI_RELEASE_FEATURES"), + "union_bins": re.findall(r"--bin ([A-Za-z0-9_-]+)", union_build), + "pq_bins": re.findall(r"--bin ([A-Za-z0-9_-]+)", pq_build), + "union_tests": tests(union_test), "release_tests": tests(rel_test), + "alone": loop.group(1).split(), + "steps": [{"step": n, "wd": wd or ".", "body": body} for n, wd, body in iv[b + 1:]], + "int_kernels_debug_features": sorted(feats(dbg)), "int_kernels_debug_tests": sorted(tests(dbg)), + "int_kernels_release_features": sorted(feats(rel)), "int_kernels_release_tests": sorted(tests(rel)), + "int_kernels_amd_features": sorted(re.search(r"^F=(\S+)$", amd, re.M).group(1).split(",")), + "int_kernels_amd_tests": sorted(tests(re.search(r'^T="(.*)"$', amd, re.M).group(1))), + "corpora_features": sorted(re.search(r"^F=(\S+)$", corp, re.M).group(1).split(",")), + "corpora": [" ".join(m.groups()) for m in re.finditer(r"^v (\S+)\s+(\S+)\s+(\S+)$", corp, re.M)], + } + + +def at_ref(ref, fn): + """fn(root) over the tree at a git ref, extracted to a scratch directory.""" + tmp = tempfile.mkdtemp(prefix="wai-lists-") + try: + arch = subprocess.run(["git", "archive", ref, ".github", "scripts", f"{CRATE}/Cargo.toml", f"{CRATE}/tests"], + capture_output=True, check=True).stdout + subprocess.run(["tar", "-x", "-C", tmp], input=arch, check=True) + return fn(tmp) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def lists_of(root): + return from_tree(root) if os.path.isdir(os.path.join(root, VERIFIERS)) else from_hand_lists(root) + + +# ------------------------------------------------- the command lines of the other steps + +# The steps outside the independent verifiers whose lists are derived: (label, job, the +# start of the step's name). Their names lost the leg's suffix when the corpus step's two +# copies, one a shell family, became one step, so they are found by the start of the name. +COMMAND_STEPS = ( + ("cargo-classes: fast integer kernels, overflow checks on", "cargo-classes", + "Fast integer kernels ≡ reference kernels (overflow checks on)"), + ("cargo-classes: fast integer kernels, release", "cargo-classes", + "Fast integer kernels ≡ reference kernels (release, overflow checks off)"), + ("cargo-classes: every cargo-class corpus", "cargo-classes", + "Every cargo-class corpus reproduces its frozen hashes"), + ("int-kernels-amd", "int-kernels-amd", "Fast integer kernels ≡ reference kernels, every kernel set"), + ("features-alone", "features-alone", "wai-rs, each feature list alone"), +) +# cargo's options that take a value, as `--opt value` or `--opt=value`. +VALUE_OPTS = {"--features", "-F", "--test", "--bin", "--example", "--bench", "--target", "-p", "--package", + "--manifest-path", "--profile", "-j", "--jobs", "--target-dir"} +SCRIPT_CALL = re.compile(r"^python3\s+\S*wai-conformance-lists\.py\s+(.*)$") + + +def tagged_args(rest): + """(tags, extra features, cargo args) of `tests|features|test-tagged TAG... [--extra-features + A,B] [-- ARGS...]`, refusing what would run no test.""" + rest = list(rest) + cargo_args = [] + if "--" in rest: + cargo_args = rest[rest.index("--") + 1:] + rest = rest[:rest.index("--")] + extra = [x for x in (opt(rest, "--extra-features") or "").split(",") if x] + unknown = [t for t in rest if t not in TAGS] + if unknown or not rest: + raise Bad(f"tags: {' '.join(unknown) or '(none given)'}; a tag is one of {', '.join(TAGS)}") + return rest, extra, cargo_args + + +def test_tagged_line(root, tags, extra, cargo_args): + names = tagged(root, tags) + if not names: + raise Bad(f"no test file carries {' or '.join(tags)}: the step would run no test") + feats = tagged_features(root, tags, extra) + return ["cargo", "test"] + cargo_args + ["--features", ",".join(feats)] + [x for t in names for x in ("--test", t)] + + +def corpus_lines(root, cs, extra): + f = ",".join(corpora_features(root, cs, extra)) + return f, [["cargo", "run", "-q", "--no-default-features", "--features", f, "--bin", c["bin"], "--", "verify", c["dir"]] + for c in cs] + + +def script_value(root, argv): + """What `$(python3 .../wai-conformance-lists.py ARGV)` prints, for the commands a step's + shell takes from it.""" + cmd, rest = argv[0], argv[1:] + if cmd in ("tests", "features"): + tags, extra, _ = tagged_args(rest) + if not tagged(root, tags): + raise Bad(f"no test file carries {' or '.join(tags)}: the step would run no test") + if cmd == "tests": + return " ".join(f"--test {t}" for t in tagged(root, tags)) + return ",".join(tagged_features(root, tags, extra)) + if cmd == "alone": + mods, errs = verifiers(root) + if errs: + raise Bad("\n".join(errs)) + return "\n".join(derive(mods)["alone"]) + raise Bad(f"`$(wai-conformance-lists.py {cmd} ...)`: the dry-run reads no value of `{cmd}`") + + +def script_commands(root, argv): + """The cargo command lines `python3 .../wai-conformance-lists.py ARGV` runs.""" + cmd, rest = argv[0], argv[1:] + if cmd == "test-tagged": + return [test_tagged_line(root, *tagged_args(rest))] + if cmd == "corpora" and "--verify" in rest: + rest = list(rest) + extra = [x for x in (opt(rest, "--extra-features") or "").split(",") if x] + cs, errs = corpora(root) + if errs: + raise Bad("\n".join(errs)) + return corpus_lines(root, cs, extra)[1] + raise Bad(f"`wai-conformance-lists.py {cmd}` in a step the dry-run reads: it reads the commands of test-tagged and corpora --verify only") + + +def subst(s, env): + """s with `$NAME` and `${NAME}` replaced from env; Bad for a name with no known value.""" + def rep(m): + name = m.group(1) or m.group(2) + if env.get(name) is None: + raise Bad(f"`${name}` has no value the dry-run knows") + return env[name] + return re.sub(r"\$\{(\w+)\}|\$(\w+)", rep, s) + + +def shell_value(rhs, env, root): + """The value an assignment's right-hand side gives, or None when it reads the runner's + environment.""" + m = re.match(r"^\$\(\s*(python3\s+\S*wai-conformance-lists\.py\s+.*?)\s*\)$", rhs) + if m: + return script_value(root, shlex.split(SCRIPT_CALL.match(m.group(1)).group(1))) + if rhs.startswith("$(") or "`" in rhs: + return None + try: + if rhs.startswith("'") and rhs.endswith("'") and len(rhs) > 1: + return rhs[1:-1] + if rhs.startswith('"') and rhs.endswith('"') and len(rhs) > 1: + return subst(rhs[1:-1], env) + return subst(rhs, env) + except Bad: + return None + + +def run_lines(lines, env, funcs, root, out): + """Append to out the cargo command lines (as argv) that a step's shell lines run: bash or + PowerShell, as far as the steps COMMAND_STEPS names use them (assignments, `for` loops, a + function a line calls, the derivations of this script, and cargo itself). A line that names + cargo but none of these is refused rather than skipped.""" + i = 0 + while i < len(lines): + line = lines[i].strip() + i += 1 + if not line or line.startswith("#"): + continue + m = re.match(r"^for (\w+) in (.*); do$", line) + if m: + depth, j = 1, i + while j < len(lines): + t = lines[j].strip() + if re.match(r"^for \w+ in .*; do$", t): + depth += 1 + elif t == "done": + depth -= 1 + if not depth: + break + j += 1 + else: + raise Bad(f"`{line}` has no `done`") + for v in shlex.split(subst(m.group(2), env)): + run_lines(lines[i:j], {**env, m.group(1): v}, funcs, root, out) + i = j + 1 + continue + m = re.match(r"^(\w+)\(\)\s*\{\s*(.*?)\s*\}$", line) + if m: + funcs[m.group(1)] = (None, m.group(2).split(";")) + continue + m = re.match(r"^function (\w+)\((.*)\)\s*\{$", line) + if m: + j = next((j for j in range(i, len(lines)) if lines[j].strip() == "}"), None) + if j is None: + raise Bad(f"`{line}` has no closing `}}`") + funcs[m.group(1)] = ([p.strip().lstrip("$") for p in m.group(2).split(",") if p.strip()], lines[i:j]) + i = j + 1 + continue + m = re.match(r"^(?:export\s+)?\$?([A-Za-z_]\w*)\s*=\s*(.*)$", line) + if m: + env[m.group(1)] = shell_value(m.group(2), env, root) + continue + word = line.split()[0] + if word in funcs: + params, body = funcs[word] + args = shlex.split(subst(line[len(word):], env)) + e = dict(env) + for n, a in enumerate(args, 1): + e[str(n)] = a + for p, a in zip(params or [], args): + e[p] = a + run_lines(body, e, funcs, root, out) + continue + m = SCRIPT_CALL.match(line) + if m: + out += script_commands(root, shlex.split(subst(m.group(1), env))) + continue + if re.search(r"(^|\s)cargo\s", line): + toks = shlex.split(subst(line, env)) + if "cargo" not in toks: + raise Bad(f"`{line}`: the dry-run cannot read this command") + out.append(toks) + continue + if "wai-conformance-lists.py" in line: + raise Bad(f"`{line}`: the dry-run cannot read this call of the script") + + +def canon(toks): + """A cargo command line in one form for every spelling of it: its flags sorted, its + features sorted and comma-joined, each target once and sorted, anything after `--` as it + was. Two commands that build and run the same thing print the same.""" + at = toks.index("cargo") + pre, sub, rest = toks[:at], toks[at + 1], toks[at + 2:] + tail = [] + if "--" in rest: + tail, rest = rest[rest.index("--"):], rest[:rest.index("--")] + flags, feats, vals = set(), set(), set() + i = 0 + while i < len(rest): + t = rest[i] + if t.startswith("--") and "=" in t and t.split("=", 1)[0] in VALUE_OPTS: + k, v = t.split("=", 1) + i += 1 + elif t in VALUE_OPTS: + if i + 1 >= len(rest): + raise Bad(f"`{' '.join(toks)}`: {t} with no value") + k, v = t, rest[i + 1] + i += 2 + else: + flags.add(t) + i += 1 + continue + if k in ("--features", "-F"): + feats |= {f for f in re.split(r"[ ,]+", v) if f} + else: + vals.add((k, v)) + out = pre + ["cargo", sub] + sorted(flags) + (["--features", ",".join(sorted(feats))] if feats else []) + out += [x for k, v in sorted(vals) for x in (k, v)] + tail + return " ".join(shlex.quote(t) for t in out) + + +def admits(cond, leg): + """Whether a step's `if:` lets it run on a leg of the family `leg` (unix or windows).""" + if cond is None: + return True + m = re.fullmatch(r"(?:\$\{\{\s*)?matrix\.os (==|!=) '([a-z0-9-]+)'(?:\s*\}\})?", cond) + if not m or m.group(2) != "windows": + raise Bad(f"a step's condition `{cond}` is none the dry-run reads (matrix.os == or != 'windows')") + return (m.group(1) == "==") == (leg == "windows") + + +def step_commands(root): + """[(label, legs, working directories, [command lines])] of the steps COMMAND_STEPS names, + each command line in canon() form; `legs` is "every leg" when the unix legs and the + Windows leg run the same, and one entry a family otherwise.""" + with open(os.path.join(root, WORKFLOW), encoding="utf-8") as fh: + text = fh.read() + out = [] + for label, job, prefix in COMMAND_STEPS: + steps = [s for s in job_steps(text, job) if s["name"].startswith(prefix)] + if not steps: + raise Bad(f"{WORKFLOW}: job `{job}` has no step `{prefix}...`") + per = {} + for leg in ("unix", "windows"): + cmds, wds = [], [] + for s in steps: + if not admits(s["if"], leg): + continue + argv = [] + try: + run_lines(joined(s["body"] or ""), {}, {}, root, argv) + except Bad as e: + raise Bad(f"{WORKFLOW}: {job} / {s['name']}: {e}") + if not argv: + raise Bad(f"{WORKFLOW}: {job} / {s['name']}: the dry-run finds no cargo command in it") + cmds += [canon(a) for a in argv] + wds.append(s["wd"] or ".") + if label == "features-alone": + cmds = sorted(cmds) + per[leg] = (uniq(wds), cmds) + if per["unix"] == per["windows"]: + out.append((label, "every leg", *per["unix"])) + else: + out += [(label, "unix legs", *per["unix"]), (label, "the Windows leg", *per["windows"])] + return out + + +# ----------------------------------------------------------------------------- output + +SETS = ("paths", "WAI_FEATURES", "WAI_PQ_FEATURES", "WAI_RELEASE_FEATURES", "union_bins", "pq_bins", + "union_tests", "release_tests", "alone", "int_kernels_debug_features", "int_kernels_debug_tests", + "int_kernels_release_features", "int_kernels_release_tests", "int_kernels_amd_features", + "int_kernels_amd_tests", "corpora_features") + + +def norm_list(k, xs): + if k == "alone": + return sorted(uniq(",".join(sorted(x.split(","))) for x in xs)) + return sorted(uniq(xs)) + + +def dry_run(L, C): + out = ["# wai-conformance.yml: what its derived lists make it run, in a stable form", + "# (lists sorted; steps and corpora in the order they run)"] + for k in SETS: + out.append(f"{k}: {' '.join(norm_list(k, L[k]))}") + out.append("corpora (in order):") + out += [f" {c}" for c in L["corpora"]] + out.append("independent-verifiers steps (in order):") + for s in L["steps"]: + out.append(f" - step: {s['step']} (working-directory: {s['wd']})") + out += [f" | {l}" for l in s["body"].split("\n")] + out.append("the other derived steps' commands (as cargo is given them: flags sorted, features sorted, each") + out.append("target once; in the order they run, but features-alone's sorted):") + for label, legs, wds, cmds in C: + out.append(f" - {label} ({legs}; working-directory: {', '.join(wds) or '-'})") + out += [f" $ {c}" for c in cmds] + return "\n".join(out) + "\n" + + +def check(root): + """Every finding about the files this script reads.""" + errs = [] + try: + mods, e = verifiers(root) + cs, e2 = corpora(root) + except Bad as x: + return [str(x)] + errs += e + e2 + targets, feats = cargo_targets(root) + tests = {n for k, n in targets if k == "test"} | {f[:-3] for f in os.listdir(os.path.join(root, CRATE, "tests")) if f.endswith(".rs")} + bins = {n for k, n in targets if k == "bin"} + # A checkout, rather than the files `dry-run --ref` extracts: directories can be checked. + full = os.path.isdir(os.path.join(root, CRATE, "src")) + gate = gate_module(root) + from pathlib import Path + wf_globs = gate.paths_of(Path(os.path.join(root, WORKFLOW))) + if sorted(gate.declared_paths(Path(os.path.join(root, VERIFIERS))) or []) != sorted(p for m in mods for p in m["paths"]): + errs.append(f"scripts/pr-touches-paths.py reads other paths from {VERIFIERS} than this script does") + try: + gate_paths(root) + except Bad as x: + errs.append(str(x)) + for m in mods: + for lst in m["features"] + (m["alone"] or []) + m["release"]: + for f in lst.split(","): + if f not in feats: + errs.append(f"{m['file']}: feature `{f}` is not in {CRATE}/Cargo.toml [features]") + for line in joined(m["body"]): + for t in re.findall(r"--test[ =]([A-Za-z0-9_-]+)", line): + if "cd wai-rs" in line and t not in tests: + errs.append(f"{m['file']}: --test {t}: {CRATE} has no such test") + for b in m["bins"] + m["pq_bins"] + re.findall(r"wai-bins/(?:union|pq)/([A-Za-z0-9_-]+)", m["body"]): + if b not in bins: + errs.append(f"{m['file']}: binary `{b}` is not a [[bin]] of {CRATE}/Cargo.toml") + for p in m["paths"]: + # A path the forge's own filter does not cover never starts the workflow, whatever + # the gate would say: a file under the glob stands for every file it matches. + sample = p.replace("**", "s/s").replace("*", "s").replace("?", "s") + if not any(gate.matches(sample, g) for g in wf_globs): + errs.append(f"{m['file']}: path `{p}` is under none of {WORKFLOW}'s paths, so the forge would never start the workflow for it") + if full and not os.path.isdir(os.path.join(root, m["wd"])): + errs.append(f"{m['file']}: working-directory `{m['wd']}` does not exist") + for c in cs: + if c["bin"] not in bins: + errs.append(f"{c['file']}: binary `{c['bin']}` is not a [[bin]] of {CRATE}/Cargo.toml") + if full and not os.path.isdir(os.path.normpath(os.path.join(root, CRATE, c["dir"]))): + errs.append(f"{c['file']}: dir `{c['dir']}` is no directory (it is read from {CRATE})") + for t, ts in markers(root).items(): + for tag in ts: + if tag not in TAGS: + errs.append(f"{CRATE}/tests/{t}.rs: `wai-conformance: {tag}` is no tag a step takes ({', '.join(TAGS)})") + return errs + + +def selftest(root, exact): + errs = check(root) + for e in errs: + print(f"check: {e}") + if errs: + return 1 + with open(os.path.join(root, BASELINE), encoding="utf-8") as fh: + base = json.load(fh) + L = from_tree(root) + added = base.get("added", {}) + bad = 0 + for k in SETS + ("corpora",): + want = norm_list(k, base[k] + added.get(k, [])) if k in SETS else base[k] + got = norm_list(k, L[k]) if k in SETS else L[k] + lost = [x for x in want if x not in got] + more = [x for x in got if x not in want] + if lost: + print(f"FAIL {k}: lost {' '.join(lost)}") + bad += 1 + elif exact and more: + print(f"FAIL {k}: more than the hand list: {' '.join(more)}") + bad += 1 + else: + print(f"ok {k}: the hand list's {len(want)}{' and ' + str(len(more)) + ' more' if more else ''}") + names = [s["step"] for s in L["steps"]] + for s in base["steps"]: + if s["step"] not in names: + print(f"FAIL step `{s['step']}` no longer runs") + bad += 1 + if exact: + # Each by name, not by place: a step or a corpus the tree adds is one addition, and + # moves no other along. The order is that of the entries both hold. + want = [s["step"] for s in base["steps"]] + for n in names: + if n not in want: + print(f"FAIL step `{n}`: more than the hand list") + bad += 1 + if [n for n in names if n in want] != [n for n in want if n in names]: + print("FAIL the steps both lists hold run in another order than the hand list's") + bad += 1 + derived = {s["step"]: s for s in L["steps"]} + for s in base["steps"]: + t = derived.get(s["step"]) + if t is not None and (s["wd"], s["body"]) != (t["wd"], t["body"]): + print(f"FAIL step `{s['step']}`: its body or directory is not the hand list's") + bad += 1 + if [c for c in L["corpora"] if c in base["corpora"]] != [c for c in base["corpora"] if c in L["corpora"]]: + print("FAIL the corpora both lists hold run in another order than the hand list's") + bad += 1 + print(f"ok {len(base['steps'])} steps of the hand list run" if not bad else f"wai-conformance-lists selftest: {bad} failure(s)") + return 1 if bad else 0 + + +# ------------------------------------------------------------------------------ main + + +def opt(args, name): + if name in args: + i = args.index(name) + v = args[i + 1] + del args[i:i + 2] + return v + return None + + +def main(argv): + args = list(argv) + root = opt(args, "--root") or os.path.dirname(HERE) + if not args: + print(__doc__) + return 2 + cmd, rest = args[0], args[1:] + try: + if cmd in ("env", "prebuild", "run", "alone", "paths"): + mods, errs = verifiers(root) + if errs: + raise Bad("\n".join(errs)) + d = derive(mods) + if cmd == "env": + for k in ("WAI_FEATURES", "WAI_PQ_FEATURES", "WAI_RELEASE_FEATURES"): + print(f"{k}={','.join(d[k])}") + elif cmd == "prebuild": + sys.stdout.write(prebuild(d)) + elif cmd == "alone": + print("\n".join(d["alone"])) + elif cmd == "paths": + print("\n".join(d["paths"])) + else: + only = [rest[i + 1] for i, a in enumerate(rest) if a == "--only"] + env = dict(os.environ) + for k in ("WAI_FEATURES", "WAI_PQ_FEATURES", "WAI_RELEASE_FEATURES"): + env[k] = ",".join(d[k]) + env.setdefault("RUNNER_TEMP", tempfile.gettempdir()) + todo = [m for m in mods if not only or m["slug"] in only] + if only and len(todo) != len(set(only)): + raise Bad(f"--only names a step no file holds: {' '.join(sorted(set(only) - {m['slug'] for m in mods}))}") + for m in todo: + print(f"::group::{m['step']} ({m['file']})", flush=True) + rc = subprocess.call(["bash", "--noprofile", "--norc", "-eo", "pipefail", os.path.join(root, m["file"])], + cwd=os.path.join(root, m["wd"]), env=env) + print("::endgroup::", flush=True) + if rc: + print(f"::error::{m['step']} failed with exit status {rc} ({m['file']})", flush=True) + return rc + print(f"every step ran and passed: {len(todo)} of {len(mods)}") + elif cmd in ("tests", "features"): + print(script_value(root, [cmd] + rest)) + elif cmd == "test-tagged": + line = test_tagged_line(root, *tagged_args(rest)) + print("+ " + " ".join(line), flush=True) + return subprocess.call(line) + elif cmd == "corpora": + extra = [x for x in (opt(rest, "--extra-features") or "").split(",") if x] + cs, errs = corpora(root) + if errs: + raise Bad("\n".join(errs)) + if "--verify" not in rest: + for c in cs: + print(f"{c['corpus']} {c['bin']} {c['dir']}") + return 0 + f, lines = corpus_lines(root, cs, extra) + print(f"the corpora's features: {f}", flush=True) + for c, line in zip(cs, lines): + print(f"::group::{c['corpus']}", flush=True) + print("+ " + " ".join(line), flush=True) + rc = subprocess.call(line) + print("::endgroup::", flush=True) + if rc: + print(f"::error::the {c['corpus']} corpus failed with exit status {rc} ({c['file']})", flush=True) + return rc + elif cmd == "dry-run": + ref = opt(rest, "--ref") + both = lambda r: (lists_of(r), step_commands(r)) + L, C = at_ref(ref, both) if ref else both(root) + sys.stdout.write(dry_run(L, C)) + elif cmd == "baseline": + ref = opt(rest, "--ref") + if not ref: + raise Bad("baseline --ref REF: the commit whose workflow kept the lists by hand") + L = at_ref(ref, from_hand_lists) + commit = subprocess.run(["git", "rev-parse", ref], capture_output=True, text=True, check=True).stdout.strip() + print(json.dumps({"commit": commit, **L}, indent=1, ensure_ascii=False)) + elif cmd == "check": + errs = check(root) + for e in errs: + print(e) + if errs: + print(f"wai-conformance-lists check: {len(errs)} finding(s)") + return 1 + print("wai-conformance-lists check: every file is well formed and names what exists.") + elif cmd == "selftest": + return selftest(root, "--exact" in rest) + else: + print(__doc__) + return 2 + except Bad as e: + print(f"wai-conformance-lists {cmd}: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/wai/CONTRIBUTING.md b/wai/CONTRIBUTING.md index 6e84b2b25..dd9deb494 100644 --- a/wai/CONTRIBUTING.md +++ b/wai/CONTRIBUTING.md @@ -56,7 +56,8 @@ not what CI runs. binary, which reports success, in any build without that feature. Declare it in `wai-rs/Cargo.toml` with a `[[test]]` stanza and `required-features`, and add `--test <name>` with the right `--features` to a step in - `.github/workflows/`. A test no workflow names runs nowhere. + `.github/workflows/`, or to a verifier step's own file (below), or tag the file for + the fast integer kernels' steps (below). A test no workflow names runs nowhere. - A test that reads inputs from outside the tree skips without them. CI sets `WAI_REQUIRE_SOTA_VECTORS=1` where it provides them, and a missing input then fails. @@ -92,6 +93,58 @@ criteria of SPEC §7, not by agreement with this implementation's own tests. A second implementation in another language is validated against the same corpora; the OER receipt kernel in C (`oer-conformance/settle/`) is one. +## Changes that add files rather than append to shared lines + +Many changes are open here at once, and a line two of them edit is a conflict that holds +one of them back. So a change adds files of its own where the tree lets it (its changelog +entry, and its lists in the conformance workflow), and in a file that stays shared it +inserts beside the entry it builds on: + +- **Its changelog entry** is a new file, `changes/<yyyy-mm-dd>-<slug>.md`, holding the + entry as the changelog does ([`changes/README.md`](changes/README.md)). It does not + edit the head of `CHANGELOG.md`. The repository checks require a change under `wai/` to + add or edit an entry file directly in `changes/` (a release, which renders the files and + removes them, passes), or to carry the trailer line `Changelog: none`, exactly, in the + trailer block of one of its commit messages. A pull request opened before the check + reached main passes as it is; one opened since that adds its entry to the head of + `CHANGELOG.md` fails. +- **A step of the independent verifiers** in `wai-conformance.yml` is a new file, + `.github/wai-conformance/verifiers/<name>.sh`: the step's shell body, run in `wai/`, + under a header of `# key: value` lines. `# step:` names it and `# order:` places it; + `# features:` brings its features into `WAI_FEATURES` and its lists into the + features-alone job; `# paths:` brings the extension text it reads into the paths the + workflow's gate runs on (the trigger takes all of `extensions/`; a path under none of the + trigger's paths still needs a line there, which `check` names); `# bins:` and + `# release-features:` cover the rest, and every `--test` it runs with `"$WAI_FEATURES"` + joins the job's first build. `scripts/wai-conformance-lists.py` + states every key; `python3 scripts/wai-conformance-lists.py dry-run` prints every list + and step the workflow runs, and `run --only <name>` runs one step. +- **A cargo-class corpus** is a new file of `.github/wai-conformance/corpora/`, naming + its tool and directory. **A test of the fast integer kernels** carries the first line + `// wai-conformance: int-kernels` (with `int-kernels-amd` when the AVX-512 host runs it + too, or `int-kernels-release` alone when only the release step does), and the steps take + its required features from `Cargo.toml`. +- **A file every change still shares** takes a new entry beside the entry it builds on, + never at the end, and is not re-sorted. `wai-rs/Cargo.toml`'s `[features]`, `[[bin]]` + and `[[test]]` tables, `wai-rs/src/lib.rs`'s `mod` lines and the tables of + `wai-rs/src/codecs/registry.rs` are grouped by topic, not sorted: a feature goes after + the feature it depends on, a target after its sibling's, a module after the module it + extends. `registry/capabilities.json`, the registry table of `REGISTRY.md` and + `vocab/v2.*` are generated from `registry.rs` in sorted order, so a new row lands at its + sorted place: never edit them by hand, and on a conflict in one take either side and run + `wai registry export` again. +- **`.gitattributes`.** A corpus whose bytes must never be converted gives its own + directory a `.gitattributes` (`* -text`), as `pin-conformance/` and + `live-conformance/scene/` do (a verifier that lists its directory's files must admit + that one, as `scene_verify.py` does), and adds no line to `wai/.gitattributes`. A + pattern only that file can hold goes beside the line it is most like, with its comment, + never at the end. +- **The corpus table of `live-conformance/README.md`**, and the list of verifier commands + under it, take a new row beside the row of the extension it builds on or extends (a + delivery record beside its delivery sibling, a live extension beside its live sibling), + never at the end. The table is written by hand: its rows name sections, schemes and + generators that no file of the corpora states. + ## Public text The specification, the extensions, READMEs, research notes, code comments and diff --git a/wai/README.md b/wai/README.md index 419005774..883cd18eb 100644 --- a/wai/README.md +++ b/wai/README.md @@ -105,8 +105,9 @@ wai conform verifier --suite claim --impl "my-verifier verdicts {corpus}" # a wai verify claim.wai-conform.json # the signed result, checked ``` -A break passes only when [`CHANGELOG.md`](CHANGELOG.md) names it: an entry this change adds that -says **Breaking** and gives the finding's id in backticks. `wai conform --list` names every kind +A break passes only when the changelog names it: an entry this change adds, as a file of +[`changes/`](changes/) or to [`CHANGELOG.md`](CHANGELOG.md), that says **Breaking** and gives the +finding's id in backticks. `wai conform --list` names every kind (verifier, decoder, packager, origin, sink) and suite, with its adapters: a command line, or an `http://` URL. `wai verify --conformant` admits a result only for a run that passed, against the corpus of the checkout it runs in. diff --git a/wai/changes/2026-11-04-no-shared-append-points.md b/wai/changes/2026-11-04-no-shared-append-points.md new file mode 100644 index 000000000..0d5649358 --- /dev/null +++ b/wai/changes/2026-11-04-no-shared-append-points.md @@ -0,0 +1,47 @@ +### Changelog entries and the conformance workflow's lists move into files a change adds + +- **Changelog entries, one a file.** A change adds its entry as a file of `changes/` + (`<yyyy-mm-dd>-<slug>.md`, holding the entry exactly as the changelog does) and leaves the + head of the changelog alone. `scripts/changelog-assemble.py --write` puts the files at the + head of `## Unreleased` for a release, newest date first and by name within a date, and + removes them. `wai breaking` reads the lines a change adds to them as it reads the lines it + adds to the changelog. The repository checks fail a pull request that changes anything + under `wai/` and adds or edits no entry file directly in `changes/` (a release, which + renders the files and removes them, passes), or one whose entry is malformed or in a + subdirectory, and fail rather than pass when they cannot compare the change with its base. + A pull request opened before the check reached main passes as it is; one opened since that + adds its entry to the head of the changelog fails, saying to add the file. The check reads + when it reached main from the base's first-parent history: the first commit there whose + tree holds it, which is the merge that brought it in. The trailer line `Changelog: none`, + exactly, in the trailer block of a commit message, says a change has nothing to record. + `changes/README.md` states the form. +- **CI: the lists `wai-conformance.yml` kept by hand are derived when it runs.** Each step of + the independent verifiers is a file of `.github/wai-conformance/verifiers/` (its body, and + a header naming its features, binaries, paths and order), each cargo-class corpus a file of + `.github/wai-conformance/corpora/`, and each test of the fast integer kernels' steps names + them in its own file (`// wai-conformance: int-kernels`). `scripts/wai-conformance-lists.py` + derives from them `WAI_FEATURES`, `WAI_PQ_FEATURES` and `WAI_RELEASE_FEATURES`, the + binaries and tests the verifiers' job builds first and the steps it runs, the feature lists + `features-alone` checks, the integer kernels' tests and features, and the corpora and their + features. A new verifier, corpus or kernel test adds a file, or a line to its own file, and + edits no line of the workflow, unless a step's `# paths:` names a file under none of the + trigger's paths. Its features and `[[test]]` entries in `wai-rs/Cargo.toml` stay shared. +- **CI: every job runs what it ran.** `.github/wai-conformance/baseline.json` holds the hand + lists the files replaced. `scripts/wai-conformance-lists.py selftest --exact` requires each + derived list to equal its hand list, and each step's body and directory to be those of the + hand list's step of its name, in the same order; the repository checks run `selftest`, + which fails a change that drops an entry of them, and a self-test that takes a step, a + feature, a tag and a corpus away in turn and requires each to fail. `dry-run` prints every + list and step in a stable form, of the tree or of a commit (`--ref`), with the command + lines of the integer kernels' steps, the corpus step, `int-kernels-amd` and the + features-alone loop, so a review compares two of them by diff alone. +- **CI: the gate reads the steps' paths.** The workflow starts on any extension text + (`wai/extensions/**`), and its `relevant` job runs the jobs only for the paths the workflow + lists and the ones the steps' files name, which it reads through + `scripts/pr-touches-paths.py --paths-from <dir> --wide <glob>`, so a new step brings the + extension text it reads in its own file. For every file outside the two paths this adds to + the gate (`.github/wai-conformance/**` and `scripts/wai-conformance-lists.py`), the gate + answers as it did. +- `wai mcp`'s conformance tools read the verifier commands from the steps' files, and + `scripts/check-tests-in-ci.py` counts a test as run when a step's file names it or a step + takes a tag it carries. diff --git a/wai/changes/README.md b/wai/changes/README.md new file mode 100644 index 000000000..2e49ce84a --- /dev/null +++ b/wai/changes/README.md @@ -0,0 +1,35 @@ +# Changelog entries, one a file + +A change to WAI adds its changelog entry here, as a file of its own, and does not edit the +head of [`../CHANGELOG.md`](../CHANGELOG.md). Two changes' entries are then two files, and +the changelog no longer makes the two changes conflict, unless both pick one name (below). + +- **The name** is `<yyyy-mm-dd>-<slug>.md`: the date the entry is written, then a few words + of its title, lowercase, joined by hyphens (`2026-11-04-pin-by-default.md`). Two changes + that pick the same name collide as files, which git reports, and one of them renames its + file. +- **The content** is exactly the entry the changelog holds, in its form: a `### ` title line, + a blank line, and the entry's text (bullets, paragraphs, `**Breaking:**` lines), with no + other heading outside a code block, and one newline at the end. +- **A break** is acknowledged here as in the changelog: `wai breaking` reads the lines a + change adds to these files as it reads the lines it adds to `CHANGELOG.md`, so a + `**Breaking:**` line naming the finding's id in backticks passes it. +- **A release** runs `python3 scripts/changelog-assemble.py --write` from the repository's + root: every entry goes to the head of `## Unreleased`, newest date first and, within a + date, by file name, and the files are removed. Without `--write` it prints the section it + would write; `--check` checks every file. +- **The check.** `scripts/check-changelog-fragment.py`, in the repository checks, fails a + pull request that changes anything under `wai/` and adds or edits no entry file here. A + file goes directly in this directory: one in a subdirectory fails, since the assembler and + `wai breaking` read only this one. A change with nothing for the changelog to say ends one + of its commit messages with the trailer line `Changelog: none`, exactly, in the message's + trailer block (its last paragraph, after a blank line, beside any other `Key: value` + trailers). A release's change, which renders these files and removes them, passes. +- **The transition.** A pull request opened before the check reached main passes as it is, + with an edit of `CHANGELOG.md` or with no entry at all. One opened since that adds its + entry to the head of `CHANGELOG.md`'s Unreleased section fails, and is told to add the file + here instead. The check reads when the pull request was opened from the event payload. It + reads when the check reached main from the first-parent history of the pull request's + base: the committer time of the first commit there whose tree holds the check, which is + the merge that brought it in. Until the check is on main, every pull request counts as + opened before it. diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs index baf5ac820..60adb6904 100644 --- a/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs @@ -17,10 +17,11 @@ //! left it does not read as the branch removing what the ref gained. //! //! Each finding names its item, its category, what changed and why that breaks; additions -//! are listed as compatible. A break is intended only when `wai/CHANGELOG.md` says so: a -//! line this change adds to it, in an entry that has the word "breaking", naming the -//! finding's id in backticks. Then it is reported as acknowledged and passes; every other -//! break fails the run (exit 1). +//! are listed as compatible. A break is intended only when the changelog says so: a line +//! this change adds to `wai/CHANGELOG.md` or to a fragment of it under `wai/changes/` (one +//! entry a file, which the release renders into the changelog), in an entry that has the +//! word "breaking", naming the finding's id in backticks. Then it is reported as +//! acknowledged and passes; every other break fails the run (exit 1). pub mod c_header; pub mod corpora; @@ -41,11 +42,30 @@ pub const OP: &str = "compat.check"; pub const USAGE: &str = "wai breaking --against <git-ref> [--category wire|record|api] [--only <surface>,...] [--skip <surface>,...]\n\ [--root <checkout>] [--target-dir <dir>] [--json|--text] what the working tree breaks against <git-ref>:\n\ - wire, record and API; a break passes only where wai/CHANGELOG.md names it (operation compat.check)"; + wire, record and API; a break passes only where wai/CHANGELOG.md or a wai/changes/ entry names it (operation compat.check)"; /// The changelog whose added lines acknowledge a break. pub const CHANGELOG: &str = "wai/CHANGELOG.md"; +/// The changelog's fragments, one entry a Markdown file, whose added lines acknowledge a break +/// as the changelog's do (`scripts/changelog-assemble.py` renders them into it). +pub const FRAGMENTS: &str = "wai/changes"; + +/// The lines this change adds to the changelog and to each of its fragments, a blank line +/// between files so that no two files' entries run together. +fn changelog_added(repo: &git::Repo, base: &str) -> Result<Vec<String>, OpError> { + let mut added = repo.added_lines(base, CHANGELOG)?; + let mut fragments: Vec<String> = std::fs::read_dir(repo.root.join(FRAGMENTS)) + .map(|rd| rd.filter_map(Result::ok).map(|e| e.file_name().to_string_lossy().into_owned()).filter(|n| n.ends_with(".md") && n != "README.md").collect()) + .unwrap_or_default(); + fragments.sort(); + for f in fragments { + added.push(String::new()); + added.extend(repo.added_lines(base, &format!("{FRAGMENTS}/{f}"))?); + } + Ok(added) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum Category { Wire, @@ -227,7 +247,7 @@ pub fn run(o: &Options) -> Result<Report, OpError> { } } findings.retain(|f| f.category <= o.category); - let acks = acknowledgements(&repo.added_lines(&base, CHANGELOG)?); + let acks = acknowledgements(&changelog_added(&repo, &base)?); for f in findings.iter_mut().filter(|f| f.breaking) { f.acknowledged = acks.contains(&f.id); } @@ -297,7 +317,7 @@ fn text(r: &Report, all: bool) { if n == 0 { println!("wai breaking: no unacknowledged breaking change"); } else { - println!("wai breaking: {n} breaking change{} not acknowledged. If {} intended, add to {CHANGELOG}, under the current version:", if n == 1 { "" } else { "s" }, if n == 1 { "it is" } else { "they are" }); + println!("wai breaking: {n} breaking change{} not acknowledged. If {} intended, say so in this change's entry, a new file of {FRAGMENTS}/ (or in {CHANGELOG}, under the current version):", if n == 1 { "" } else { "s" }, if n == 1 { "it is" } else { "they are" }); for f in breaks.iter().filter(|f| !f.acknowledged).take(8) { println!(" - **Breaking:** `{}`: <why it is intended, and what a user does about it>", f.id); } diff --git a/wai/wai-rs/src/bin/wai_cli/mcp/tools.rs b/wai/wai-rs/src/bin/wai_cli/mcp/tools.rs index 5b9e50283..82a67cdcf 100644 --- a/wai/wai-rs/src/bin/wai_cli/mcp/tools.rs +++ b/wai/wai-rs/src/bin/wai_cli/mcp/tools.rs @@ -789,10 +789,17 @@ fn package_build(server: &mut Server, args: &Value) -> Result<Value, String> { // ---------------------------------------------------------------------------- conformance /// Every (verifier, corpus) pair: the `python3 [-I] <script.py> <dir>` lines of the conformance -/// workflow, when the tree holds it, and each `*-conformance/*_verify.py` with its directory. +/// workflow and of its verifier steps' files (`.github/wai-conformance/verifiers/`), when the +/// tree holds them, and each `*-conformance/*_verify.py` with its directory. fn corpora(root: &Path) -> Vec<(String, String)> { let mut pairs = std::collections::BTreeSet::new(); - if let Ok(wf) = std::fs::read_to_string(root.join("../.github/workflows/wai-conformance.yml")) { + let mut texts: Vec<String> = std::fs::read_to_string(root.join("../.github/workflows/wai-conformance.yml")).into_iter().collect(); + if let Ok(rd) = std::fs::read_dir(root.join("../.github/wai-conformance/verifiers")) { + let mut steps: Vec<PathBuf> = rd.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "sh")).collect(); + steps.sort(); + texts.extend(steps.iter().filter_map(|p| std::fs::read_to_string(p).ok())); + } + for wf in &texts { for line in wf.lines() { let t = line.trim().trim_start_matches("run:").trim(); let Some(rest) = t.strip_prefix("python3 ") else { continue }; diff --git a/wai/wai-rs/tests/cli_breaking.rs b/wai/wai-rs/tests/cli_breaking.rs index cbc3170d8..9e2c27301 100644 --- a/wai/wai-rs/tests/cli_breaking.rs +++ b/wai/wai-rs/tests/cli_breaking.rs @@ -1,6 +1,7 @@ //! `wai breaking` on a small repository laid out as this one is: each kind of break the gate //! exists for is reported, fails the run, and passes only once a line the change adds to -//! `wai/CHANGELOG.md` names it in an entry that says "breaking": +//! `wai/CHANGELOG.md`, or to a fragment of it under `wai/changes/`, names it in an entry that +//! says "breaking": //! //! - a registered code removed (a capability, a wire identifier's magic); //! - a record's signing preimage changed (its domain-separation prefix; and its signed @@ -161,6 +162,42 @@ fn a_removed_registered_code_breaks_until_acknowledged() { reset(&d); } +/// A fragment of the changelog (a file of `wai/changes/`, one entry) acknowledges a break as a +/// line added to the changelog does; a fragment the base already holds acknowledges nothing, +/// and neither does the fragments' README. +#[test] +fn a_changelog_fragment_acknowledges_a_break() { + let (d, base) = repo("fragment"); + let id = "wire:capability:wai.test.removed"; + write(&d, "wai/registry/capabilities.json", ®ISTRY.replace( + r#", + {"capability": "wai.test.removed", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}"#, + "", + )); + assert_eq!(breaking(&d, &base, &["--only", "registry"]).0, 1, "the removal breaks"); + + write(&d, "wai/changes/README.md", &format!("An example entry:\n\n- **Breaking:** `{id}`: an example.\n")); + assert_eq!(breaking(&d, &base, &["--only", "registry"]).0, 1, "the README is no entry"); + write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- Renamed `{id}`.\n")); + assert_eq!(breaking(&d, &base, &["--only", "registry"]).0, 1, "an entry without the word acknowledges nothing"); + write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- **Breaking:** `{id}`: intended, for the test.\n")); + let (code, v) = breaking(&d, &base, &["--only", "registry"]); + assert_eq!(code, 0, "acknowledged by the fragment: {v:#}"); + assert_eq!(finding(&v, id).unwrap()["acknowledged"], true); + + // Committed into the base, the same fragment acknowledges nothing a later change breaks. + git(&d, &["add", "-A"]); + git(&d, &["commit", "-q", "-m", "the removal and its entry"]); + let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); + let base2 = String::from_utf8_lossy(&o.stdout).trim().to_string(); + write(&d, "wai/registry/capabilities.json", ®ISTRY.replace(r#"{"name": "WTS1", "kind": "magic", "document": "extensions/t.md", "section": "§2", "status": "defined"},"#, "")); + write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- **Breaking:** `{id}`: intended, for the test.\n- **Breaking:** `wire:wire-identifier:magic/WTS1`, said in the base.\n")); + let (code, v) = breaking(&d, &base2, &["--only", "registry"]); + assert_eq!(code, 0, "a line added to a fragment the base holds acknowledges: {v:#}"); + write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- **Breaking:** `{id}`: intended, for the test.\n")); + assert_eq!(breaking(&d, &base2, &["--only", "registry"]).0, 1, "the base's fragment alone acknowledges nothing new"); +} + #[test] fn a_changed_signing_preimage_breaks_until_acknowledged() { let (d, base) = repo("preimage"); diff --git a/wai/wai-rs/tests/cli_generate_score.rs b/wai/wai-rs/tests/cli_generate_score.rs index 3a5d44a1f..c2cb690f9 100644 --- a/wai/wai-rs/tests/cli_generate_score.rs +++ b/wai/wai-rs/tests/cli_generate_score.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai generate` and `wai score build|render`, run as a user runs them: the native //! pins a page checks its own output against. Each output is checked against a //! committed vector — `tests/vectors/generate/` for pictures, `score-conformance/` diff --git a/wai/wai-rs/tests/cli_stage.rs b/wai/wai-rs/tests/cli_stage.rs index 654d822df..4725ab7a3 100644 --- a/wai/wai-rs/tests/cli_stage.rs +++ b/wai/wai-rs/tests/cli_stage.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai stage`: the tool writes, applies, inspects, packs and explodes //! staged-delivery layers (`extensions/staged-delivery.md`; SPEC §7.1), and `wai wrap` //! refuses a refinement capability. These run the binary on the committed corpus diff --git a/wai/wai-rs/tests/generate.rs b/wai/wai-rs/tests/generate.rs index f891df790..042c39e1c 100644 --- a/wai/wai-rs/tests/generate.rs +++ b/wai/wai-rs/tests/generate.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.generate.spectral` — **generation-equivalence**: a deterministic integer //! generator produces byte-identical bytes on every machine from the same //! `(model, prompt, seed, steps)`, so its hash is portable (the create-side of diff --git a/wai/wai-rs/tests/int_audio.rs b/wai/wai-rs/tests/int_audio.rs index 2c2c9dd1a..ecb2197a6 100644 --- a/wai/wai-rs/tests/int_audio.rs +++ b/wai/wai-rs/tests/int_audio.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.audio.int_codec` — **sample-equivalence**: a LEARNED 1-D audio transform //! codec whose integer synthesis (transposed convs + IGDN) decodes a factorized //! rANS latent to int16 PCM byte-identically on every machine, so the PCM hash is diff --git a/wai/wai-rs/tests/int_audio_ops.rs b/wai/wai-rs/tests/int_audio_ops.rs index fb41f9b65..2c67315b4 100644 --- a/wai/wai-rs/tests/int_audio_ops.rs +++ b/wai/wai-rs/tests/int_audio_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.audio.int_codec` through the integer operator set: its lowering `wam1/1` //! (`extensions/integer-operators.md` §13.3, with the PCM step) and the certificate of //! every committed set (§8.5), as this crate loads them (`int_ops`). diff --git a/wai/wai-rs/tests/int_hyper_ops.rs b/wai/wai-rs/tests/int_hyper_ops.rs index 9ec31cd0c..7f303acbb 100644 --- a/wai/wai-rs/tests/int_hyper_ops.rs +++ b/wai/wai-rs/tests/int_hyper_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.neural.int_hyper` and the keyframe of `wai.video.int_hyper` through the integer //! operator set: their lowering `whm1/1` (`extensions/integer-operators.md` §13.2; `h_s`, //! then `g_s` with the RGB8 step of §13.0) and the certificate of every committed set diff --git a/wai/wai-rs/tests/int_kernels_geometry.rs b/wai/wai-rs/tests/int_kernels_geometry.rs index f715e60aa..21a85eb05 100644 --- a/wai/wai-rs/tests/int_kernels_geometry.rs +++ b/wai/wai-rs/tests/int_kernels_geometry.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! Geometry at the edges of what the fast integer kernels lay out, run through the //! public API under the scalar reference kernels AND under each fast-kernel policy //! (serial, explicit multi-threaded tilings, automatic): diff --git a/wai/wai-rs/tests/int_kernels_vectors.rs b/wai/wai-rs/tests/int_kernels_vectors.rs index 46105341c..f6d3b58af 100644 --- a/wai/wai-rs/tests/int_kernels_vectors.rs +++ b/wai/wai-rs/tests/int_kernels_vectors.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! Every committed integer-decoder vector whose decode reaches the fast kernels //! (int_hyper, int_mlicpp, int_audio, int_splat, int_world one and two agents, the //! neural-keyframe video, the int_synth corpus), plus the real-footage video, diff --git a/wai/wai-rs/tests/int_mlicpp.rs b/wai/wai-rs/tests/int_mlicpp.rs index 8ae58c971..caaecac98 100644 --- a/wai/wai-rs/tests/int_mlicpp.rs +++ b/wai/wai-rs/tests/int_mlicpp.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.neural.int_mlicpp` on its committed conformance vector //! (`tests/vectors/int_mlicpp/`, written by `tools/wai_int_mlicpp_export.py`). No //! test here skips: the vector is in the repository. diff --git a/wai/wai-rs/tests/int_mlicpp_ops.rs b/wai/wai-rs/tests/int_mlicpp_ops.rs index dd843ec51..0c378ef65 100644 --- a/wai/wai-rs/tests/int_mlicpp_ops.rs +++ b/wai/wai-rs/tests/int_mlicpp_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.neural.int_mlicpp` through the integer operator set: its lowering `wmb1/1` //! (`extensions/integer-operators.md` §13.4; each of the bundle's three `WHM1` networks by //! `whm1/1`, §13.2: `h_s`, `entropy`, then `g_s` with the RGB8 step of §13.0) and the diff --git a/wai/wai-rs/tests/int_mlicpp_preflight.rs b/wai/wai-rs/tests/int_mlicpp_preflight.rs index ded630379..53d77d45b 100644 --- a/wai/wai-rs/tests/int_mlicpp_preflight.rs +++ b/wai/wai-rs/tests/int_mlicpp_preflight.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! A `wai.neural.int_mlicpp` payload whose header the model cannot fit is refused //! before the decode allocates anything sized from it. //! diff --git a/wai/wai-rs/tests/int_ops_conformance.rs b/wai/wai-rs/tests/int_ops_conformance.rs index f8a1d2ff4..8b4b7529d 100644 --- a/wai/wai-rs/tests/int_ops_conformance.rs +++ b/wai/wai-rs/tests/int_ops_conformance.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! Every graph vector of `ops-conformance/` (`vectors/` and `graphs/`: each operator on //! its own, and the multi-operator graphs, the lowerings and the state vectors among them) //! decoded through the integer operator set's interpreter with this crate's fast kernels diff --git a/wai/wai-rs/tests/int_ops_differential.rs b/wai/wai-rs/tests/int_ops_differential.rs index aba629f87..0bc91796f 100644 --- a/wai/wai-rs/tests/int_ops_differential.rs +++ b/wai/wai-rs/tests/int_ops_differential.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! Each integer decoder through the integer operator set against its direct decode //! (`legacy_int`, test-only), on random parameter sets and payloads. The two must give //! the same output, or both refuse with the same code. The only inputs on which they may diff --git a/wai/wai-rs/tests/int_payload.rs b/wai/wai-rs/tests/int_payload.rs index beda78f74..f1fda7601 100644 --- a/wai/wai-rs/tests/int_payload.rs +++ b/wai/wai-rs/tests/int_payload.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! The payloads of the WAI-native integer codecs (`extensions/integer-payloads.md`) //! against the committed corpus `wai/int-payload-conformance/`: //! diff --git a/wai/wai-rs/tests/int_payload_cost.rs b/wai/wai-rs/tests/int_payload_cost.rs index 329b4fa6a..b7feb6b12 100644 --- a/wai/wai-rs/tests/int_payload_cost.rs +++ b/wai/wai-rs/tests/int_payload_cost.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! The declared cost of an integer decode (`int_payload::declared_cost`, //! `extensions/integer-payloads.md` §7, *Memory and work*) against what the decode //! holds, measured with a global allocator that tracks the bytes live and their peak. diff --git a/wai/wai-rs/tests/int_payload_preflight.rs b/wai/wai-rs/tests/int_payload_preflight.rs index 68c0d4951..dcad2b911 100644 --- a/wai/wai-rs/tests/int_payload_preflight.rs +++ b/wai/wai-rs/tests/int_payload_preflight.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! An integer payload whose header asks for more than the caps allow, or more than its //! parameter set can fit, is refused before the decode allocates anything sized from //! it (`extensions/integer-payloads.md` §7). diff --git a/wai/wai-rs/tests/int_set_cache.rs b/wai/wai-rs/tests/int_set_cache.rs index 40019715b..0d9dbd5f7 100644 --- a/wai/wai-rs/tests/int_set_cache.rs +++ b/wai/wai-rs/tests/int_set_cache.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! The store of loaded parameter sets (`int_set_cache`) under the decodes that use it, //! `int_payload::decode_int` and `declared_cost`, for every capability that decodes //! against a set through the integer operator set. diff --git a/wai/wai-rs/tests/int_splat.rs b/wai/wai-rs/tests/int_splat.rs index 6ae4c2620..b0a336e26 100644 --- a/wai/wai-rs/tests/int_splat.rs +++ b/wai/wai-rs/tests/int_splat.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.splat.int_codec` — **splat-equivalence**: a LEARNED Gaussian-splat attribute //! codec whose integer synthesis decodes a factorized-rANS latent to the `[14,S,S]` //! attribute tensor byte-identically on every machine, so the reconstructed cloud's diff --git a/wai/wai-rs/tests/int_splat_ops.rs b/wai/wai-rs/tests/int_splat_ops.rs index d2b9fa76f..4c777c270 100644 --- a/wai/wai-rs/tests/int_splat_ops.rs +++ b/wai/wai-rs/tests/int_splat_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.splat.int_codec` through the integer operator set: its lowering `whm1/1` //! (`extensions/integer-operators.md` §13.2, with the RGB8 step of §13.0) and the //! certificate of every committed set (§8.5), as this crate loads them (`int_ops`). diff --git a/wai/wai-rs/tests/int_synth_ops.rs b/wai/wai-rs/tests/int_synth_ops.rs index d95569675..2ad0ee7bd 100644 --- a/wai/wai-rs/tests/int_synth_ops.rs +++ b/wai/wai-rs/tests/int_synth_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.neural.int_synth` through the integer operator set: its lowering `int_synth/1` //! (`extensions/integer-operators.md` §13.1) and the certificate of every committed //! set (§8.5, §14.3), as this crate loads them (`int_ops`). diff --git a/wai/wai-rs/tests/int_video.rs b/wai/wai-rs/tests/int_video.rs index 65fd24fe0..51992b734 100644 --- a/wai/wai-rs/tests/int_video.rs +++ b/wai/wai-rs/tests/int_video.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.video.int_motion` — **frame-state-equivalence**: a deterministic integer //! inter-frame codec reconstructs every frame byte-identically on every machine, //! so each frame's hash (and the whole clip's) is portable. Committed vector under diff --git a/wai/wai-rs/tests/int_video_enc_memory.rs b/wai/wai-rs/tests/int_video_enc_memory.rs index a9bd93ab5..bdb21c797 100644 --- a/wai/wai-rs/tests/int_video_enc_memory.rs +++ b/wai/wai-rs/tests/int_video_enc_memory.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! What the integer video encoder holds (`int_video_enc::working_bytes`) against what it //! holds in fact, measured with a global allocator that tracks the bytes live and their //! peak, as `tests/int_payload_cost.rs` measures the decoder. diff --git a/wai/wai-rs/tests/int_video_halfpel.rs b/wai/wai-rs/tests/int_video_halfpel.rs index db5f7ea73..63a4f9e1b 100644 --- a/wai/wai-rs/tests/int_video_halfpel.rs +++ b/wai/wai-rs/tests/int_video_halfpel.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.video.int_motion` with **HALF-PEL** motion compensation — the decoder's //! `predict()` reconstructs sub-pixel block predictions by integer bilinear //! interpolation (`(a+b+1)>>1`, `(a+b+c+d+2)>>2`), gated on the bitstream's diff --git a/wai/wai-rs/tests/int_video_keyframe.rs b/wai/wai-rs/tests/int_video_keyframe.rs index 7aa97dae7..64694f315 100644 --- a/wai/wai-rs/tests/int_video_keyframe.rs +++ b/wai/wai-rs/tests/int_video_keyframe.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.video.int_hyper` — the NEURAL-KEYFRAME inter-frame codec: the I-frame is //! decoded by the learned integer codec `int_hyper`, the P-frames are integer //! block-MC residuals on top. The whole decode is integer, so every frame is diff --git a/wai/wai-rs/tests/int_video_ops.rs b/wai/wai-rs/tests/int_video_ops.rs index 9b6c7916d..541b0e4fc 100644 --- a/wai/wai-rs/tests/int_video_ops.rs +++ b/wai/wai-rs/tests/int_video_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `WIV1` (`wai.video.int_motion`, and the P-frames of `wai.video.int_hyper`) through the //! integer operator set: the stage operators of `extensions/integer-operators.md` §13.5, //! E1 `rans64_decode`, E10 `mc_predict` and E9 `residual_step`, with no graph. diff --git a/wai/wai-rs/tests/int_world.rs b/wai/wai-rs/tests/int_world.rs index 197748895..d759e4bf8 100644 --- a/wai/wai-rs/tests/int_world.rs +++ b/wai/wai-rs/tests/int_world.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.world.model` integer renderer — a generative INTERACTIVE world whose every //! frame is byte-exact. The committed transition + synthesis models, an initial //! latent, and an action walk regenerate a frame sequence whose per-frame and clip diff --git a/wai/wai-rs/tests/int_world2.rs b/wai/wai-rs/tests/int_world2.rs index 4c74bbafa..0658ce4ef 100644 --- a/wai/wai-rs/tests/int_world2.rs +++ b/wai/wai-rs/tests/int_world2.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! `wai.world.model` TWO-AGENT co-control — both agents' actions drive one frame. //! The committed transition + synthesis (A=4 input planes: dxA,dyA,dxB,dyB), an //! initial latent, and a sequence of combined (agent-A, agent-B) actions regenerate a diff --git a/wai/wai-rs/tests/int_world_memory.rs b/wai/wai-rs/tests/int_world_memory.rs index db84803ec..668c4bbd2 100644 --- a/wai/wai-rs/tests/int_world_memory.rs +++ b/wai/wai-rs/tests/int_world_memory.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! What `wai.world.model`'s integer renderer holds, through the integer operator set, //! against the direct renderer it replaced (`int_world::legacy`, feature `legacy_int`), //! measured with a global allocator that tracks the bytes live and their peak. diff --git a/wai/wai-rs/tests/int_world_ops.rs b/wai/wai-rs/tests/int_world_ops.rs index 9d36982c6..7084bc55c 100644 --- a/wai/wai-rs/tests/int_world_ops.rs +++ b/wai/wai-rs/tests/int_world_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.world.model`'s integer renderer through the integer operator set: its lowering //! `world/1` (`extensions/integer-operators.md` §13.5, informative: the transition `T` //! with its `A` action planes, then the synthesis `g_s` with the RGB8 step of §13.0) and diff --git a/wai/wai-rs/tests/jpegai_bitstream_calibration.rs b/wai/wai-rs/tests/jpegai_bitstream_calibration.rs index 08b359b4e..3da5edccf 100644 --- a/wai/wai-rs/tests/jpegai_bitstream_calibration.rs +++ b/wai/wai-rs/tests/jpegai_bitstream_calibration.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! Calibration for [`wai::jpegai_bitstream`] against a REAL JPEG-AI codestream //! (`test256.bin`): walk the marker segments and check the layout, then replay the //! picture header's leading fields with the bit reader and match the reference values. diff --git a/wai/wai-rs/tests/jpegai_nets_ops.rs b/wai/wai-rs/tests/jpegai_nets_ops.rs index e6078f3d8..9bb7b792f 100644 --- a/wai/wai-rs/tests/jpegai_nets_ops.rs +++ b/wai/wai-rs/tests/jpegai_nets_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.image.jpegai`'s other integer networks through the integer operator set: the //! hyper decoder, the four context stages and the synthesis, as `jpegai-int/1` lowers //! them (`extensions/integer-operators.md` §13.6.2) to the graphs `rec_y`, `rec_uv`, diff --git a/wai/wai-rs/tests/jpegai_ops.rs b/wai/wai-rs/tests/jpegai_ops.rs index 0e97e1147..23baaa3fb 100644 --- a/wai/wai-rs/tests/jpegai_ops.rs +++ b/wai/wai-rs/tests/jpegai_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `wai.image.jpegai`'s scale path and entropy decodes through the integer operator set: //! its lowering `jpegai-int/1` (`extensions/integer-operators.md` §13.6.1) and the //! certificate of each bundle (§8.5), as this crate loads them (`jpegai_ops`). diff --git a/wai/wai-rs/tests/manifest_parse_memory.rs b/wai/wai-rs/tests/manifest_parse_memory.rs index 36950ed2c..a84a6a8a2 100644 --- a/wai/wai-rs/tests/manifest_parse_memory.rs +++ b/wai/wai-rs/tests/manifest_parse_memory.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels-release //! What reading a manifest allocates (SPEC §2, *Reader limits*; SECURITY.md, *Manifest //! parsing*). A counting global allocator records every byte the reader asks for, and //! each case requires the peak to stay within the bound the reference states: 128 bytes diff --git a/wai/wai-rs/tests/manifest_reader_equivalence.rs b/wai/wai-rs/tests/manifest_reader_equivalence.rs index 921b31d7a..348336614 100644 --- a/wai/wai-rs/tests/manifest_reader_equivalence.rs +++ b/wai/wai-rs/tests/manifest_reader_equivalence.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels-release //! The reference readers build a manifest with `container::read_manifest` and //! `read_manifest_v2`, which move its members out of one JSON tree (SPEC §2, *Reader //! limits*: so a parse allocates within a bound), where they used serde's derived reader diff --git a/wai/wai-rs/tests/stage_measure.rs b/wai/wai-rs/tests/stage_measure.rs index 08391aa09..323145f90 100644 --- a/wai/wai-rs/tests/stage_measure.rs +++ b/wai/wai-rs/tests/stage_measure.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! Staged-delivery measurement against the committed corpora //! (`extensions/staged-measurement.md`): the byte rule over every committed envelope, //! the refusal of text forms, and a title read and evaluated from a map of files, as a diff --git a/wai/wai-rs/tests/stage_measure_bounds.rs b/wai/wai-rs/tests/stage_measure_bounds.rs index 98f4ba9ff..75e749e4b 100644 --- a/wai/wai-rs/tests/stage_measure_bounds.rs +++ b/wai/wai-rs/tests/stage_measure_bounds.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! A title whose base declares an outsized decode is refused before anything is //! allocated for it (`extensions/staged-measurement.md` Appendix D): the measurement //! decodes a base only through `staged::decode_base_pinned`, under the title's cost diff --git a/wai/wai-rs/tests/staged_resources.rs b/wai/wai-rs/tests/staged_resources.rs index 8cd7240b6..f88b6fd04 100644 --- a/wai/wai-rs/tests/staged_resources.rs +++ b/wai/wai-rs/tests/staged_resources.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels //! The staged sink's own work and memory (`extensions/staged-delivery.md` §4.5 and //! Appendix A.5), measured with a global allocator that counts, for each thread, the //! bytes it allocated, its allocations, the bytes live and their peak. The counts are diff --git a/wai/wai-rs/tests/wir1_ops.rs b/wai/wai-rs/tests/wir1_ops.rs index 159b9d221..dc47bc61a 100644 --- a/wai/wai-rs/tests/wir1_ops.rs +++ b/wai/wai-rs/tests/wir1_ops.rs @@ -1,3 +1,4 @@ +// wai-conformance: int-kernels int-kernels-amd //! `WIR1` (`wai.{image,video,audio,splat}.int_refine`, staged-delivery Appendix A) //! through the integer operator set: the stage operators of //! `extensions/integer-operators.md` §13.5, E1 `rans64_decode`, E11 `up2_predict`, E12