From ef4d02e54712b57ec3ec49466b101c9809766312 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 5 Nov 2026 08:03:56 -0500 Subject: [PATCH 1/5] wai-quantum-writer: WAI Quantum writes a program for a task, on its own model The loop the job service will write programs with, as a crate of its own. The task is an exercise in the course's form (wai.quantum.exercise/1). A model writes a program, and the engine checks it against the task exactly as it checks a lesson's answer (engine::course::check_exercise). What went wrong goes back to the model in the engine's own words, until a program passes or the rounds run out. - The code model (src/coder.rs): North Mini Code 1.0 (Apache-2.0) on the pure-Rust runtime (jouleclaw-loader-gguf). - Decoding is greedy, so the same model files, task and engine give the same programs. - The conversation stays in the model's cache: a repair round feeds only the new turn, where the measurement harness re-read the whole conversation each round. - The loop takes the answer after the model's thinking, so thinking text is never checked as a program, and a reply that never leaves its thinking is asked again. - The model's identity: the digest of a manifest of its files (wai.quantum.model/1, wai-model.json). open() checks each file's size against it, and verify reads every byte. - The transcript (wai.quantum.write/1): the task, model and prompt (version, system and first-turn digests), the decoding, the engine's version, and each attempt (program, digest, the engine's check, tokens). Timing is returned beside it, since a replay does not reproduce time. - The engine: - course::task_of shows any exercise as a lesson shows its own, its title and task given as one string or one per locale; - course::exercise gives a lesson's exercise document; - lesson() now builds its fields through the same helpers, and a test holds every lesson's fields in both locales equal to task_of's. - The command line: wai-quantum-writer write | course | manifest | verify. - CI: wai-quantum-cli.yml tests the writer with a scripted model, with and without the model compiled in, plus clippy. The workflow now also runs for changes to the writer and to the loader crate it depends on. Tests (all run on the build machines): - wai-quantum-writer: 7 pass, default features and without them; clippy -D warnings is clean. - engine::course: 11 pass. - scripts/check-lockfiles.sh: 61 lockfiles, all current. Stacked on #645 (check_exercise). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-cli.yml | 18 + wai/CHANGELOG.md | 15 + wai/wai-quantum-writer/Cargo.lock | 707 ++++++++++++++++++++++++++ wai/wai-quantum-writer/Cargo.toml | 27 + wai/wai-quantum-writer/src/coder.rs | 205 ++++++++ wai/wai-quantum-writer/src/lib.rs | 232 +++++++++ wai/wai-quantum-writer/src/main.rs | 100 ++++ wai/wai-quantum-writer/tests/write.rs | 110 ++++ wai/wai-quantum/src/engine/course.rs | 123 +++-- 9 files changed, 1505 insertions(+), 32 deletions(-) create mode 100644 wai/wai-quantum-writer/Cargo.lock create mode 100644 wai/wai-quantum-writer/Cargo.toml create mode 100644 wai/wai-quantum-writer/src/coder.rs create mode 100644 wai/wai-quantum-writer/src/lib.rs create mode 100644 wai/wai-quantum-writer/src/main.rs create mode 100644 wai/wai-quantum-writer/tests/write.rs diff --git a/.github/workflows/wai-quantum-cli.yml b/.github/workflows/wai-quantum-cli.yml index 8c2bc37e3..c2f3d0de7 100644 --- a/.github/workflows/wai-quantum-cli.yml +++ b/.github/workflows/wai-quantum-cli.yml @@ -21,6 +21,9 @@ on: - 'wai/wai-quantum/**' - 'wai/wai-quantum-cli/**' - 'wai/wai-quantum-qpu/**' + - 'wai/wai-quantum-writer/**' + # The writer's model runs on this crate, which it path-depends on. + - 'jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/**' - 'wai/quantum-conformance/eval/**' - 'rust-toolchain.toml' - '.github/workflows/wai-quantum-cli.yml' @@ -29,6 +32,9 @@ on: - 'wai/wai-quantum/**' - 'wai/wai-quantum-cli/**' - 'wai/wai-quantum-qpu/**' + - 'wai/wai-quantum-writer/**' + # The writer's model runs on this crate, which it path-depends on. + - 'jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/**' - 'wai/quantum-conformance/eval/**' - 'rust-toolchain.toml' - '.github/workflows/wai-quantum-cli.yml' @@ -103,6 +109,18 @@ jobs: python3 -I -c 'import ast, sys; ast.parse(open(sys.argv[1]).read())' ../quantum-conformance/eval/token_cost.py cargo clippy --locked --release --all-targets -- -D warnings cargo clippy --locked --release --all-targets --no-default-features -- -D warnings + # wai-quantum-writer: the loop that writes a program for a task, its model on the + # pure-Rust runtime, checked by the engine as a lesson's answer is. Its tests run the + # loop with a scripted model (no checkpoint), with and without the model compiled in. + - name: The writer, with a scripted model, and clippy + shell: bash + working-directory: wai/wai-quantum-writer + run: | + set -euo pipefail + cargo test --locked --release + cargo test --locked --release --no-default-features + cargo clippy --locked --release --all-targets -- -D warnings + cargo clippy --locked --release --all-targets --no-default-features -- -D warnings - name: The smoke checks shell: bash working-directory: wai/wai-quantum-cli diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 6306a85b1..08b20029e 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -152,6 +152,21 @@ a task that is not in the course is held to what it must do, the first step towa for an intent and checking each attempt. `check` now looks up its lesson and calls it, and its answers are the same bytes. +### wai-quantum-writer: WAI Quantum writes a program for a task, on its own model + +A new crate, `wai/wai-quantum-writer`, holds the loop the job service will write programs with. +- **The task:** an exercise in the course's form (`wai.quantum.exercise/1`), shown to the model through + `engine::course::task_of`. `task_of` and `engine::course::exercise` are new; `lesson` shows the same fields through it. +- **The loop:** the model writes a program, the engine checks it as it checks a lesson's answer, and what went wrong + goes back for the next round, until a program passes or the rounds run out. +- **The model:** North Mini Code 1.0 (Apache-2.0) on the pure-Rust runtime (`jouleclaw-loader-gguf`). Decoding is + greedy, and the conversation is held in the model's cache, so a repair round reads only what is new. The loop takes + only the answer that follows the model's thinking. +- **The transcript** (`wai.quantum.write/1`) names the task, the model and the prompt by digest (the model by the + digest of its file manifest, `wai.quantum.model/1`), and each attempt by its program's digest and the engine's check. + The same model files, task and engine give the same bytes. +- **The command line:** `wai-quantum-writer write`, `course`, `manifest` and `verify`. + ### Contribution links: a packet taken after its sequence number's drop point is late, whatever the timer did - **[live-contribution](extensions/live-contribution.md) §6.1:** a packet *arrives* when the receiver takes it. An diff --git a/wai/wai-quantum-writer/Cargo.lock b/wai/wai-quantum-writer/Cargo.lock new file mode 100644 index 000000000..682f671ce --- /dev/null +++ b/wai/wai-quantum-writer/Cargo.lock @@ -0,0 +1,707 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cc" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jouleclaw-core" +version = "0.1.0" + +[[package]] +name = "jouleclaw-loader-gguf" +version = "0.1.0" +dependencies = [ + "fancy-regex", + "jouleclaw-core", + "libc", + "memmap2", + "quick-xml", + "rayon", + "regex", + "serde", + "serde_json", + "thiserror", + "windows-sys", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" +dependencies = [ + "libc", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "foldhash", + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wai-quantum" +version = "0.8.1" +dependencies = [ + "blake3", + "ed25519-dalek", + "serde", + "serde_json", +] + +[[package]] +name = "wai-quantum-writer" +version = "0.1.0" +dependencies = [ + "blake3", + "jouleclaw-loader-gguf", + "serde_json", + "wai-quantum", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/wai/wai-quantum-writer/Cargo.toml b/wai/wai-quantum-writer/Cargo.toml new file mode 100644 index 000000000..d6371aad1 --- /dev/null +++ b/wai/wai-quantum-writer/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "wai-quantum-writer" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +description = "WAI Quantum writes a program for a task: a model on WAI's own pure-Rust inference runtime writes it, the engine checks it against what the task asks exactly as it checks a lesson's answer, and what went wrong goes back for another attempt. Decoding is greedy, so the same model files, task and engine give the same programs, and every attempt is in a transcript named by digests." +repository = "https://git.transaction.science/Transaction-Science/open-standards" +publish = false + +[dependencies] +# quantum_engine: the course's check of a program against an exercise +# (engine::course::check_exercise, task_of); quantum_receipt: the canonical +# bytes the transcript's digests are taken over. +wai-quantum = { path = "../wai-quantum", default-features = false, features = ["quantum_engine", "quantum_receipt"] } +serde_json = { version = "1", features = ["preserve_order"] } +blake3 = "1" +# The model the loop runs: North Mini Code 1.0 (Apache-2.0) on the pure-Rust +# inference substrate (jouleclaw-loader-gguf), greedy decode from the checkpoint mmap. +jouleclaw-loader-gguf = { path = "../../jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf", features = ["rayon"], optional = true } + +[features] +default = ["coder"] +coder = ["dep:jouleclaw-loader-gguf"] + +[[bin]] +name = "wai-quantum-writer" +required-features = ["coder"] diff --git a/wai/wai-quantum-writer/src/coder.rs b/wai/wai-quantum-writer/src/coder.rs new file mode 100644 index 000000000..34813053d --- /dev/null +++ b/wai/wai-quantum-writer/src/coder.rs @@ -0,0 +1,205 @@ +//! The model the job service writes with: North Mini Code 1.0 (Cohere2-MoE, 30B total and +//! 3B active, Apache-2.0), on WAI's own pure-Rust inference runtime, decoded greedily from +//! the checkpoint's memory map. +//! +//! One conversation is held in the model's cache: [`Model::open`] reads the first prompt +//! once, and each later turn feeds only what is new, so a repair round does not read the +//! whole conversation again. The model thinks before it answers (`<|START_THINKING|>` … +//! `<|END_THINKING|>`); the loop is given only the answer. +//! +//! The model is named by the manifest of its files, `wai-model.json` beside them ([`manifest`]): +//! each file's name, size and BLAKE3 digest. Its digest is the digest of that manifest, so a +//! transcript names exactly the weights that wrote its programs. [`CodeModel::open`] refuses a +//! directory whose files do not have the sizes the manifest states; [`verify`] reads every +//! byte. + +use crate::{Model, Reply}; +use jouleclaw_loader_gguf::bpe_tokenizer::BpeTokenizer; +use jouleclaw_loader_gguf::cohere2_moe::{Cohere2MoeConfig, Cohere2MoeModel, DecodeState}; +use jouleclaw_loader_gguf::safetensors_reader::read_safetensors_file; +use serde_json::{json, Value}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +/// The manifest's form, and its file's name. +pub const MANIFEST_SCHEMA: &str = "wai.quantum.model/1"; +pub const MANIFEST_FILE: &str = "wai-model.json"; +/// What the manifest says the model is. +pub const NAME: &str = "North Mini Code 1.0"; +pub const LICENCE: &str = "Apache-2.0"; + +const START_THINKING: &str = "<|START_THINKING|>"; +const END_THINKING: &str = "<|END_THINKING|>"; + +fn turn(role: &str, text: &str) -> String { + format!("<|START_OF_TURN_TOKEN|><|{role}_TOKEN|>{text}<|END_OF_TURN_TOKEN|>") +} + +/// The answer in a reply: the text after the thinking, the whole reply when the model did +/// not think, and None while the thinking has not ended. +pub fn answer_of(reply: &str) -> Option<&str> { + match (reply.find(START_THINKING), reply.find(END_THINKING)) { + (_, Some(end)) => Some(&reply[end + END_THINKING.len()..]), + (Some(_), None) => None, + (None, None) => Some(reply), + } +} + +/// The files that make up the model, in the order the manifest lists them. +fn model_files(dir: &Path) -> Result, String> { + let mut files: Vec = std::fs::read_dir(dir) + .map_err(|e| format!("{}: {e}", dir.display()))? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + let n = p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + n == "config.json" || n == "tokenizer.json" || (n.starts_with("model") && n.ends_with(".safetensors")) + }) + .collect(); + files.sort(); + if !files.iter().any(|p| p.ends_with("config.json")) || !files.iter().any(|p| p.ends_with("tokenizer.json")) { + return Err(format!("{}: not a model directory (config.json and tokenizer.json are needed)", dir.display())); + } + Ok(files) +} + +fn file_digest(p: &Path) -> Result { + let mut f = std::fs::File::open(p).map_err(|e| format!("{}: {e}", p.display()))?; + let mut h = blake3::Hasher::new(); + let mut buf = vec![0u8; 1 << 24]; + loop { + let n = f.read(&mut buf).map_err(|e| format!("{}: {e}", p.display()))?; + if n == 0 { + break; + } + h.update(&buf[..n]); + } + Ok(h.finalize().to_hex().to_string()) +} + +/// The manifest of the model in `dir`, every file read and digested. +pub fn manifest(dir: &Path) -> Result { + let files = model_files(dir)? + .iter() + .map(|p| { + let size = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len(); + Ok(json!({ "name": p.file_name().unwrap().to_string_lossy(), "size": size, "blake3": file_digest(p)? })) + }) + .collect::, String>>()?; + Ok(json!({ "schema": MANIFEST_SCHEMA, "name": NAME, "licence": LICENCE, "files": files })) +} + +/// Read the manifest in `dir` and check that each file it lists is there with its size. +fn read_manifest(dir: &Path) -> Result { + let path = dir.join(MANIFEST_FILE); + let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e} (write it with `wai-quantum-writer manifest {}`)", path.display(), dir.display()))?; + let m: Value = serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?; + if m["schema"] != MANIFEST_SCHEMA { + return Err(format!("{}: not a {MANIFEST_SCHEMA} manifest", path.display())); + } + for f in m["files"].as_array().into_iter().flatten() { + let name = f["name"].as_str().unwrap_or_default(); + let size = std::fs::metadata(dir.join(name)).map(|m| m.len()).map_err(|e| format!("{name}: {e}"))?; + if Some(size) != f["size"].as_u64() { + return Err(format!("{name}: {size} bytes, the manifest says {}", f["size"])); + } + } + Ok(m) +} + +/// Read every byte of the model in `dir` and compare it with its manifest. +pub fn verify(dir: &Path) -> Result<(), String> { + let stated = read_manifest(dir)?; + let now = manifest(dir)?; + if stated["files"] == now["files"] { + Ok(()) + } else { + Err(format!("{}: the files do not match their manifest", dir.display())) + } +} + +/// North Mini Code 1.0, bound to its checkpoint, holding one conversation. +pub struct CodeModel { + model: Cohere2MoeModel, + tok: BpeTokenizer, + identity: Value, + bos: u32, + stops: Vec, + state: Option, + logits: Vec, + /// A token chosen and kept in the reply but not yet fed (the reply stopped on it). + pending: Option, +} + +impl CodeModel { + /// Bind the checkpoint in `dir` (memory-mapped, nothing copied) after checking it + /// against its manifest's sizes. + pub fn open(dir: &Path) -> Result { + let m = read_manifest(dir)?; + let config: Cohere2MoeConfig = serde_json::from_slice(&std::fs::read(dir.join("config.json")).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?; + let shards = model_files(dir)? + .into_iter() + .filter(|p| p.extension().is_some_and(|e| e == "safetensors")) + .map(|p| read_safetensors_file(&p).map_err(|e| format!("{}: {e:?}", p.display()))) + .collect::, String>>()?; + let model = Cohere2MoeModel::bind_sharded(config, shards).map_err(|e| format!("{e:?}"))?; + let tok = BpeTokenizer::from_file(dir.join("tokenizer.json")).map_err(|e| format!("{e:?}"))?; + let stops = ["<|END_OF_TURN_TOKEN|>", ""].iter().filter_map(|s| tok.special_id(s)).collect(); + let bos = tok.special_id("").ok_or("the tokenizer has no ")?; + let identity = json!({ "name": m["name"], "licence": m["licence"], "digest": crate::digest(&m), "runtime": "jouleclaw-loader-gguf cohere2_moe" }); + Ok(CodeModel { model, tok, identity, bos, stops, state: None, logits: Vec::new(), pending: None }) + } + + fn feed(&mut self, ids: &[u32]) -> Result<(), String> { + let state = self.state.as_mut().ok_or("no conversation is open")?; + for &id in ids { + self.logits = self.model.forward_token(id, state); + } + Ok(()) + } +} + +impl Model for CodeModel { + fn identity(&self) -> Value { + self.identity.clone() + } + + fn open(&mut self, system: &str, user: &str) -> Result<(), String> { + let mut ids = vec![self.bos]; + ids.extend(self.tok.encode(&format!("{}{}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", turn("SYSTEM", system), turn("USER", user)))); + let mut state = self.model.new_state(); + self.logits = self.model.forward_prompt(&ids, &mut state); + self.state = Some(state); + self.pending = None; + Ok(()) + } + + fn reply(&mut self, max_new: usize, done: &dyn Fn(&str) -> bool) -> Result { + let mut ids: Vec = Vec::new(); + for i in 0..max_new { + // Greedy: the highest logit, the lowest token id on a tie. + let next = self.logits.iter().enumerate().fold((0usize, f32::NEG_INFINITY), |(bi, bv), (i, &v)| if v > bv { (i, v) } else { (bi, bv) }).0 as u32; + if self.stops.contains(&next) { + break; + } + ids.push(next); + if i % 4 == 3 { + let so_far = self.tok.decode(&ids); + if answer_of(&so_far).is_some_and(done) { + self.pending = Some(next); + break; + } + } + self.feed(&[next])?; + } + let text = self.tok.decode(&ids); + let answer = answer_of(&text).map(str::to_string); + let answer_tokens = answer.as_deref().map(|a| self.tok.encode(a).len()).unwrap_or(0).min(ids.len()); + Ok(Reply { answer, tokens: ids.len(), thinking_tokens: ids.len() - answer_tokens }) + } + + fn answer(&mut self, user: &str) -> Result<(), String> { + let mut ids: Vec = self.pending.take().into_iter().collect(); + ids.extend(self.tok.encode(&format!("<|END_OF_TURN_TOKEN|>{}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", turn("USER", user)))); + self.feed(&ids) + } +} diff --git a/wai/wai-quantum-writer/src/lib.rs b/wai/wai-quantum-writer/src/lib.rs new file mode 100644 index 000000000..afaaf4ab9 --- /dev/null +++ b/wai/wai-quantum-writer/src/lib.rs @@ -0,0 +1,232 @@ +//! WAI Quantum writes a program for a task. +//! +//! A task is an exercise in the course's form (`wai.quantum.exercise/1`): what the +//! program must do (an intent, or one for each case), its constraints, the job it runs +//! as, and a starter program. A model writes a program for it. The engine checks that +//! program against the task exactly as it checks a lesson's answer +//! ([`course::check_exercise`]), and when the check fails, what went wrong goes back to +//! the model in the engine's own words for another attempt. The loop stops at the first +//! program that passes, or after its rounds. +//! +//! Decoding is greedy, so the same model files, task, options and engine give the same +//! programs, round for round: anyone who holds the model can replay a transcript and get +//! its bytes. The transcript ([`SCHEMA`]) names the model by the digest of its files, +//! the prompt by its version and digest, and each attempt by its program's digest and +//! the engine's check. How long each step took is returned beside it ([`Written::timing`]), +//! not inside it, since time is the one thing a replay does not reproduce. + +use serde_json::{json, Value}; +use wai_quantum::engine::{course, receipt, schema}; + +#[cfg(feature = "coder")] +pub mod coder; + +/// The transcript's form. +pub const SCHEMA: &str = "wai.quantum.write/1"; +/// The version of the prompt: the system text, how a task is put to the model, and how +/// a failed check is put back. A change to any of them is a new version. +pub const PROMPT_VERSION: &str = "wai.quantum.write.prompt/1"; +/// What the model is told before every task. +pub const SYSTEM: &str = "You write OpenQASM 3 programs. Reply with exactly one complete OpenQASM 3 program in a ```qasm code block, and nothing after it. Start it with `OPENQASM 3.0;` and `include \"stdgates.inc\";`."; +/// What the model is told when its reply ended before it wrote a program. +pub const NO_ANSWER: &str = "Your reply ended before you wrote the program. Think briefly, then write the complete program."; + +/// How the loop runs. +#[derive(Clone, Debug)] +pub struct Options { + /// Attempts after the first, each with the engine's account of the last one. + pub rounds: usize, + /// The most tokens one reply may take, its thinking included. + pub max_new: usize, + /// The memory one check's run may take, in bytes. + pub job_bytes: u64, +} + +impl Default for Options { + fn default() -> Options { + Options { rounds: 4, max_new: 3000, job_bytes: 1 << 30 } + } +} + +/// One reply of the model's. +#[derive(Clone, Debug, Default)] +pub struct Reply { + /// The answer: the text after the model's thinking, or the whole reply when it did + /// not think. None when the reply ended inside its thinking. + pub answer: Option, + /// Every token of the reply, its thinking included. + pub tokens: usize, + /// The tokens of the reply that were thinking. + pub thinking_tokens: usize, +} + +/// A model the loop asks: one conversation at a time, continued turn by turn, decoded +/// greedily. +pub trait Model { + /// What names the model in a transcript: its name, its licence, and the digest of + /// its files. + fn identity(&self) -> Value; + /// Start a conversation with the system text and the first turn; the model's reply + /// comes next. + fn open(&mut self, system: &str, user: &str) -> Result<(), String>; + /// The model's reply, until it ends its turn, `max_new` tokens have passed, or + /// `done` says the answer so far is enough. + fn reply(&mut self, max_new: usize, done: &dyn Fn(&str) -> bool) -> Result; + /// Continue the conversation with the next turn, after the model's last reply. + fn answer(&mut self, user: &str) -> Result<(), String>; +} + +/// A transcript, and the seconds each attempt took (its reply, then its check). +#[derive(Clone, Debug)] +pub struct Written { + pub transcript: Value, + pub timing: Vec<(f64, f64)>, +} + +/// The digest of a document: BLAKE3 over its canonical bytes, as the engine's receipts +/// take it. +pub fn digest(v: &Value) -> String { + blake3::hash(&receipt::canonical(v)).to_hex().to_string() +} + +/// The digest of a text: BLAKE3 over its UTF-8 bytes. +pub fn text_digest(s: &str) -> String { + blake3::hash(s.as_bytes()).to_hex().to_string() +} + +/// The program in an answer: its first fenced code block, else the text from +/// `OPENQASM` on, else the answer itself. +pub fn program_of(answer: &str) -> String { + for fence in ["```qasm", "```openqasm", "```"] { + if let Some(at) = answer.find(fence) { + let body = &answer[at + fence.len()..]; + let body = body.strip_prefix('\n').unwrap_or(body); + let end = body.find("```").unwrap_or(body.len()); + return body[..end].trim().to_string() + "\n"; + } + } + match answer.find("OPENQASM") { + Some(at) => answer[at..].trim().to_string() + "\n", + None => answer.trim().to_string() + "\n", + } +} + +/// Whether an answer's first code block has closed: then the program is complete, and +/// nothing the model would write after it is read. +pub fn block_closed(answer: &str) -> bool { + answer.find("```").is_some_and(|open| answer[open + 3..].contains("```")) +} + +fn lines(v: &Value) -> String { + match v { + Value::String(s) => s.clone(), + Value::Array(a) => a.iter().filter_map(Value::as_str).map(|l| format!("{l}\n")).collect(), + _ => String::new(), + } +} + +/// How a task is put to the model: its title and task, what the program must do in the +/// engine's words, and the starter program when it has one. +pub fn prompt(task: &Value) -> String { + let t = course::task_of(task, "en"); + let mut s = String::new(); + if let Some(title) = t["title"].as_str().filter(|x| !x.is_empty()) { + s.push_str(&format!("Task: {title}\n\n")); + } + if let Some(text) = t["task"].as_str().filter(|x| !x.is_empty()) { + s.push_str(&format!("{text}\n\n")); + } + let reqs: Vec<&str> = t["requirements"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + if !reqs.is_empty() { + s.push_str("The program must do this, as the engine checks it:\n"); + for r in reqs { + s.push_str(&format!("- {r}\n")); + } + s.push('\n'); + } + let starter = lines(&t["starter"]); + if !starter.is_empty() { + s.push_str(&format!("Starter program:\n```qasm\n{starter}```\n")); + } + s.trim_end().to_string() +} + +/// How a failed check is put back to the model: each thing that went wrong, in the +/// course's words and the engine's own, and where to look first. +pub fn feedback(check: &Value) -> String { + let mut s = String::from("The engine checked that program and it does not do what the task asks yet:\n\n"); + for w in check["what_went_wrong"].as_array().into_iter().flatten() { + s.push_str(&format!("- {}\n", w["text"].as_str().unwrap_or(""))); + if let Some(e) = w["engine"].as_str() { + s.push_str(&format!(" the engine: {e}\n")); + } + } + if let Some(p) = check["pointer"].as_str() { + s.push_str(&format!("\nWhere to look: {p}\n")); + } + s.push_str("\nWrite the corrected complete program."); + s +} + +/// Write a program for `task`: the model writes, the engine checks, and what went wrong +/// goes back, until a program passes or the rounds run out. +pub fn write(model: &mut dyn Model, task: &Value, opts: &Options) -> Result { + if !task.is_object() || (task.get("intent").is_none() && task.get("cases").is_none()) { + return Err("a task is an exercise (wai.quantum.exercise/1) with an intent, or cases that each have one".into()); + } + let first = prompt(task); + model.open(SYSTEM, &first)?; + let mut attempts = Vec::new(); + let mut timing = Vec::new(); + let mut passed_round = None; + for round in 0..=opts.rounds { + let t = std::time::Instant::now(); + let reply = model.reply(opts.max_new, &block_closed)?; + let reply_secs = t.elapsed().as_secs_f64(); + let t = std::time::Instant::now(); + let program = reply.answer.as_deref().map(program_of); + let check = match &program { + Some(p) => course::check_exercise(task, p, opts.job_bytes, "en")?, + None => Value::Null, + }; + timing.push((reply_secs, t.elapsed().as_secs_f64())); + let passed = check["passed"] == true; + attempts.push(json!({ + "round": round, + "answered": program.is_some(), + "program": program, + "program_digest": program.as_deref().map(text_digest), + "passed": passed, + "level": check["level"], + "check": check, + "reply_tokens": reply.tokens, + "thinking_tokens": reply.thinking_tokens, + })); + if passed { + passed_round = Some(round); + break; + } + if round == opts.rounds { + break; + } + let next = if program.is_some() { feedback(&check) } else { NO_ANSWER.to_string() }; + model.answer(&next)?; + } + let last = attempts.last().cloned().unwrap_or(Value::Null); + let transcript = json!({ + "schema": SCHEMA, + "task": task, + "task_digest": digest(task), + "model": model.identity(), + "prompt": { "version": PROMPT_VERSION, "system_digest": text_digest(SYSTEM), "first_digest": text_digest(&first) }, + "decoding": { "method": "greedy", "max_new": opts.max_new, "rounds": opts.rounds }, + "engine": schema::ENGINE_VERSION, + "passed": passed_round.is_some(), + "passed_round": passed_round, + "level": if passed_round.is_some() { last["level"].clone() } else { Value::Null }, + "program": last["program"], + "program_digest": last["program_digest"], + "attempts": attempts, + }); + Ok(Written { transcript, timing }) +} diff --git a/wai/wai-quantum-writer/src/main.rs b/wai/wai-quantum-writer/src/main.rs new file mode 100644 index 000000000..237ff29eb --- /dev/null +++ b/wai/wai-quantum-writer/src/main.rs @@ -0,0 +1,100 @@ +//! `wai-quantum-writer`: WAI Quantum writes a program for a task, on this machine. +//! +//! ```text +//! wai-quantum-writer write --model DIR TASK.json [--rounds N] [--max-new N] # the transcript, as JSON +//! wai-quantum-writer course --model DIR --out DIR [--rounds N] [--max-new N] [--lessons a,b] +//! wai-quantum-writer manifest DIR # write DIR/wai-model.json: every file's size and digest +//! wai-quantum-writer verify DIR # read every byte of the model against its manifest +//! ``` +//! +//! `course` writes every exercise of the course "Quantum circuits, checked" and records, in +//! `OUT/results.jsonl`, each lesson's transcript with the seconds each attempt took; it prints +//! how many passed after each round of repair. + +use serde_json::{json, Value}; +use std::io::Write; +use std::path::PathBuf; +use wai_quantum::engine::course; +use wai_quantum_writer::{coder, write, Options}; + +fn flag(args: &[String], name: &str) -> Option { + args.iter().position(|a| a == name).and_then(|i| args.get(i + 1).cloned()) +} + +fn options(args: &[String]) -> Result { + let mut o = Options::default(); + if let Some(r) = flag(args, "--rounds") { + o.rounds = r.parse().map_err(|_| "--rounds: a number")?; + } + if let Some(m) = flag(args, "--max-new") { + o.max_new = m.parse().map_err(|_| "--max-new: a number")?; + } + Ok(o) +} + +fn run(args: &[String]) -> Result<(), String> { + match args.first().map(String::as_str) { + Some("manifest") => { + let dir = PathBuf::from(args.get(1).ok_or("manifest DIR")?); + let m = coder::manifest(&dir)?; + let path = dir.join(coder::MANIFEST_FILE); + std::fs::write(&path, serde_json::to_string_pretty(&m).unwrap() + "\n").map_err(|e| format!("{}: {e}", path.display()))?; + println!("{}: {} files, digest {}", path.display(), m["files"].as_array().map_or(0, Vec::len), wai_quantum_writer::digest(&m)); + Ok(()) + } + Some("verify") => { + let dir = PathBuf::from(args.get(1).ok_or("verify DIR")?); + coder::verify(&dir)?; + println!("{}: every file matches its manifest", dir.display()); + Ok(()) + } + Some("write") => { + let dir = PathBuf::from(flag(args, "--model").ok_or("write --model DIR TASK.json")?); + let file = args.iter().skip(1).find(|a| a.ends_with(".json")).ok_or("write --model DIR TASK.json")?; + let task: Value = serde_json::from_str(&std::fs::read_to_string(file).map_err(|e| format!("{file}: {e}"))?).map_err(|e| format!("{file}: {e}"))?; + let mut model = coder::CodeModel::open(&dir)?; + let w = write(&mut model, &task, &options(args)?)?; + println!("{}", serde_json::to_string_pretty(&w.transcript).unwrap()); + for (i, (reply, check)) in w.timing.iter().enumerate() { + eprintln!("round {i}: reply {reply:.0} s, check {check:.1} s"); + } + Ok(()) + } + Some("course") => { + let dir = PathBuf::from(flag(args, "--model").ok_or("course --model DIR --out DIR")?); + let out = PathBuf::from(flag(args, "--out").ok_or("course --model DIR --out DIR")?); + let only: Option> = flag(args, "--lessons").map(|s| s.split(',').map(str::to_string).collect()); + let opts = options(args)?; + std::fs::create_dir_all(&out).map_err(|e| e.to_string())?; + let mut model = coder::CodeModel::open(&dir)?; + let mut results = std::fs::File::create(out.join("results.jsonl")).map_err(|e| e.to_string())?; + let ids: Vec<&str> = course::lesson_ids()?.into_iter().filter(|id| only.as_ref().is_none_or(|o| o.iter().any(|x| x == id))).collect(); + let mut passed_at: Vec> = Vec::new(); + for id in &ids { + let w = write(&mut model, &course::exercise(id)?, &opts)?; + let at = w.transcript["passed_round"].as_u64(); + eprintln!("{id}: {}", at.map_or("never passed".to_string(), |r| format!("passed after {r} repair(s)"))); + passed_at.push(at); + writeln!(results, "{}", json!({ "lesson": id, "timing": w.timing, "transcript": w.transcript })).map_err(|e| e.to_string())?; + results.flush().map_err(|e| e.to_string())?; + } + let n = ids.len(); + for r in 0..=opts.rounds as u64 { + let k = passed_at.iter().filter(|p| p.is_some_and(|p| p <= r)).count(); + println!("passed after {r} repair(s): {k}/{n}"); + } + let never: Vec<&&str> = ids.iter().zip(&passed_at).filter(|(_, p)| p.is_none()).map(|(l, _)| l).collect(); + println!("never passed: {never:?}"); + Ok(()) + } + _ => Err("usage: wai-quantum-writer write --model DIR TASK.json | course --model DIR --out DIR | manifest DIR | verify DIR".into()), + } +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + if let Err(e) = run(&args) { + eprintln!("wai-quantum-writer: {e}"); + std::process::exit(1); + } +} diff --git a/wai/wai-quantum-writer/tests/write.rs b/wai/wai-quantum-writer/tests/write.rs new file mode 100644 index 000000000..e4388372c --- /dev/null +++ b/wai/wai-quantum-writer/tests/write.rs @@ -0,0 +1,110 @@ +//! The loop, with a scripted model: its replies set in advance, what it is told recorded. +//! No checkpoint is needed, so these run anywhere the engine builds. + +use serde_json::{json, Value}; +use wai_quantum_writer::{block_closed, program_of, prompt, write, Model, Options, Reply, NO_ANSWER, SCHEMA}; + +struct Scripted { + replies: Vec>, + told: Vec, +} + +impl Model for Scripted { + fn identity(&self) -> Value { + json!({ "name": "scripted", "licence": "none", "digest": "0" }) + } + fn open(&mut self, system: &str, user: &str) -> Result<(), String> { + self.told = vec![system.to_string(), user.to_string()]; + Ok(()) + } + fn reply(&mut self, _max_new: usize, _done: &dyn Fn(&str) -> bool) -> Result { + let next = self.replies.remove(0); + Ok(Reply { answer: next.map(str::to_string), tokens: 10, thinking_tokens: 4 }) + } + fn answer(&mut self, user: &str) -> Result<(), String> { + self.told.push(user.to_string()); + Ok(()) + } +} + +const BELL: &str = "```qasm\nOPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[2] q;\nbit[2] c;\nh q[0];\ncx q[0], q[1];\nc = measure q;\n```"; +const NO_CNOT: &str = "```qasm\nOPENQASM 3.0;\ninclude \"stdgates.inc\";\nqubit[2] q;\nbit[2] c;\nh q[0];\nc = measure q;\n```"; + +fn bell_task() -> Value { + json!({ + "schema": "wai.quantum.exercise/1", + "title": "A Bell pair", + "task": "Entangle two qubits so that their measured bits always agree.", + "starter": ["OPENQASM 3.0;", "include \"stdgates.inc\";", "qubit[2] q;", "bit[2] c;", "c = measure q;"], + "intent": { "distribution": { "00": 0.5, "11": 0.5 } } + }) +} + +#[test] +fn a_program_that_passes_at_once_ends_the_loop() { + let mut m = Scripted { replies: vec![Some(BELL)], told: vec![] }; + let w = write(&mut m, &bell_task(), &Options::default()).unwrap(); + let t = &w.transcript; + assert_eq!(t["schema"], SCHEMA); + assert_eq!((t["passed"].as_bool(), t["passed_round"].as_u64(), t["level"].as_str()), (Some(true), Some(0), Some("proof")), "{t:#}"); + assert_eq!(t["attempts"].as_array().unwrap().len(), 1); + assert!(t["program"].as_str().unwrap().contains("cx q[0], q[1];")); + // The task went to the model in words, with its starter program. + assert!(m.told[1].contains("A Bell pair") && m.told[1].contains("00 with probability 0.5") && m.told[1].contains("qubit[2] q;"), "{}", m.told[1]); +} + +#[test] +fn what_went_wrong_goes_back_in_the_engines_words_and_the_repair_passes() { + let mut m = Scripted { replies: vec![Some(NO_CNOT), Some(BELL)], told: vec![] }; + let w = write(&mut m, &bell_task(), &Options::default()).unwrap(); + let t = &w.transcript; + assert_eq!(t["passed_round"].as_u64(), Some(1), "{t:#}"); + assert_eq!(t["attempts"][0]["passed"], false); + assert_eq!(t["attempts"][0]["check"]["what_went_wrong"][0]["kind"], "distribution"); + let told = &m.told[2]; + assert!(told.starts_with("The engine checked that program"), "{told}"); + assert!(told.contains("the engine:") && told.contains("Where to look:"), "{told}"); + assert_eq!(w.timing.len(), 2); +} + +#[test] +fn a_reply_that_never_leaves_its_thinking_is_no_answer_and_is_asked_again() { + let mut m = Scripted { replies: vec![None, Some(BELL)], told: vec![] }; + let w = write(&mut m, &bell_task(), &Options::default()).unwrap(); + let t = &w.transcript; + assert_eq!((t["attempts"][0]["answered"].as_bool(), t["attempts"][0]["check"].is_null()), (Some(false), true), "{t:#}"); + assert_eq!(m.told[2], NO_ANSWER); + assert_eq!(t["passed_round"].as_u64(), Some(1)); +} + +#[test] +fn the_rounds_run_out_without_a_pass() { + let mut m = Scripted { replies: vec![Some(NO_CNOT); 3], told: vec![] }; + let w = write(&mut m, &bell_task(), &Options { rounds: 2, ..Options::default() }).unwrap(); + let t = &w.transcript; + assert_eq!((t["passed"].as_bool(), t["passed_round"].is_null(), t["level"].is_null()), (Some(false), true, true), "{t:#}"); + assert_eq!(t["attempts"].as_array().unwrap().len(), 3); +} + +#[test] +fn the_same_model_task_and_options_give_the_same_transcript_bytes() { + let run = || { + let mut m = Scripted { replies: vec![Some(NO_CNOT), Some(BELL)], told: vec![] }; + serde_json::to_vec(&write(&mut m, &bell_task(), &Options::default()).unwrap().transcript).unwrap() + }; + assert_eq!(run(), run()); +} + +#[test] +fn a_task_needs_something_the_program_must_do() { + let mut m = Scripted { replies: vec![], told: vec![] }; + assert!(write(&mut m, &json!({ "task": "anything" }), &Options::default()).is_err()); +} + +#[test] +fn the_program_is_the_answers_code_block() { + assert_eq!(program_of("Here:\n```qasm\nOPENQASM 3.0;\nqubit q;\n```\nDone."), "OPENQASM 3.0;\nqubit q;\n"); + assert_eq!(program_of("OPENQASM 3.0;\nqubit q;"), "OPENQASM 3.0;\nqubit q;\n"); + assert!(block_closed("```qasm\nx\n```") && !block_closed("```qasm\nx\n")); + assert!(prompt(&bell_task()).starts_with("Task: A Bell pair")); +} diff --git a/wai/wai-quantum/src/engine/course.rs b/wai/wai-quantum/src/engine/course.rs index 64ff0822e..4d5124565 100644 --- a/wai/wai-quantum/src/engine/course.rs +++ b/wai/wai-quantum/src/engine/course.rs @@ -519,38 +519,8 @@ pub fn lesson(id: &str, loc: &str) -> Result { let l = &c.lessons[i]; let ex = &l.exercise; let unit = &c.manifest["units"][l.unit]; - let cases: Vec = ex - .get("cases") - .and_then(Value::as_array) - .into_iter() - .flatten() - .map(|k| { - let mut v = json!({ "name": word(&k["name"], loc), "inputs": k["inputs"] }); - if let Some(i) = k.get("intent") { - v["intent"] = public_intents(i, loc); - } - v - }) - .collect(); - let mut requirements: Vec = Vec::new(); - if let Some(i) = ex.get("intent") { - requirements.extend(public_intents(i, loc).as_array().into_iter().flatten().filter_map(|x| x["text"].as_str().map(str::to_string))); - } - for k in &cases { - if let Some(texts) = k["intent"].as_array() { - let t: Vec<&str> = texts.iter().filter_map(|x| x["text"].as_str()).collect(); - requirements.push(format!("{}: {}", k["name"].as_str().unwrap_or(""), t.join("; "))); - } - } - if let Some(n) = ex.pointer("/job/noise").and_then(Value::as_object) { - let parts: Vec = n.iter().map(|(k, p)| format!("{k} = {}", num(p.as_f64().unwrap_or(0.0)))).collect(); - requirements.push(say(loc, "intent.noise", &[("noise", parts.join(", "))])); - } - if !cases.is_empty() { - let names: Vec<&str> = cases.iter().filter_map(|k| k["name"].as_str()).collect(); - requirements.push(say(loc, "intent.cases", &[("cases", names.join(if loc == "zh-CN" { "；" } else { "; " }))])); - } - requirements.extend(constraint_texts(ex.get("constraints").unwrap_or(&Value::Null), loc).into_iter().map(|(_, _, t)| t)); + let cases = cases_view(ex, loc); + let requirements = requirements_of(ex, &cases, loc); let hints: Vec<&str> = ex["hints"][loc].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); let mut v = json!({ "id": l.id, @@ -578,6 +548,69 @@ pub fn lesson(id: &str, loc: &str) -> Result { Ok(v) } +/// An exercise's cases as a reader is shown them: each name, its inputs, and +/// the words of its intents. +fn cases_view(ex: &Value, loc: &str) -> Vec { + ex.get("cases") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|k| { + let mut v = json!({ "name": word(&k["name"], loc), "inputs": k["inputs"] }); + if let Some(i) = k.get("intent") { + v["intent"] = public_intents(i, loc); + } + v + }) + .collect() +} + +/// What an exercise asks of a program, in words: each intent, each case's, +/// the noise its job runs under, its cases, and each constraint. +fn requirements_of(ex: &Value, cases: &[Value], loc: &str) -> Vec { + let mut requirements: Vec = Vec::new(); + if let Some(i) = ex.get("intent") { + requirements.extend(public_intents(i, loc).as_array().into_iter().flatten().filter_map(|x| x["text"].as_str().map(str::to_string))); + } + for k in cases { + if let Some(texts) = k["intent"].as_array() { + let t: Vec<&str> = texts.iter().filter_map(|x| x["text"].as_str()).collect(); + requirements.push(format!("{}: {}", k["name"].as_str().unwrap_or(""), t.join("; "))); + } + } + if let Some(n) = ex.pointer("/job/noise").and_then(Value::as_object) { + let parts: Vec = n.iter().map(|(k, p)| format!("{k} = {}", num(p.as_f64().unwrap_or(0.0)))).collect(); + requirements.push(say(loc, "intent.noise", &[("noise", parts.join(", "))])); + } + if !cases.is_empty() { + let names: Vec<&str> = cases.iter().filter_map(|k| k["name"].as_str()).collect(); + requirements.push(say(loc, "intent.cases", &[("cases", names.join(if loc == "zh-CN" { "；" } else { "; " }))])); + } + requirements.extend(constraint_texts(ex.get("constraints").unwrap_or(&Value::Null), loc).into_iter().map(|(_, _, t)| t)); + requirements +} + +/// Any exercise, as a lesson shows its own (see [`lesson`]): its title and +/// task in `loc`, its starter program, what it asks in words, its cases, job +/// and constraints. A task a program is to be written for is shown so. Its +/// title and task may be one string as well as one per locale. +pub fn task_of(exercise: &Value, loc: &str) -> Value { + let loc = locale(loc); + let cases = cases_view(exercise, loc); + let requirements = requirements_of(exercise, &cases, loc); + json!({ + "locale": loc, + "title": exercise["title"].as_str().unwrap_or_else(|| word(&exercise["title"], loc)), + "task": exercise["task"].as_str().unwrap_or_else(|| word(&exercise["task"], loc)), + "starter": exercise["starter"], + "intent": public_intents(exercise.get("intent").unwrap_or(&Value::Null), loc), + "cases": cases, + "job": exercise.get("job").cloned().unwrap_or(json!({})), + "constraints": exercise.get("constraints").cloned().unwrap_or(json!({})), + "requirements": requirements, + }) +} + /// One thing a check found wrong, in the locale, with where to look. struct Wrong { kind: &'static str, @@ -697,6 +730,14 @@ pub fn check(id: &str, src: &str, job_bytes: u64, loc: &str) -> Result Result { + let (c, i) = find(id)?; + Ok(c.lessons[i].exercise.clone()) +} + /// A program checked against any exercise in the form [`EXERCISE_SCHEMA`] /// names, as [`check`] checks a lesson's: the same verdicts, constraints and /// words, with `lesson` null. It is how a task that is not in the course (a @@ -1047,6 +1088,24 @@ mod tests { } } + #[test] + fn any_exercise_is_shown_as_a_lesson_shows_its_own() { + for id in lesson_ids().unwrap() { + for loc in LOCALES { + let (c, i) = find(id).unwrap(); + let shown = lesson(id, loc).unwrap(); + let task = task_of(&c.lessons[i].exercise, loc); + for k in ["locale", "title", "task", "starter", "intent", "cases", "job", "constraints", "requirements"] { + assert_eq!(task[k], shown[k], "{id} {loc} {k}"); + } + } + } + let mine = json!({ "schema": EXERCISE_SCHEMA, "title": "A Bell pair", "task": "Entangle two qubits.", "intent": { "distribution": { "00": 0.5, "11": 0.5 } } }); + let t = task_of(&mine, "en"); + assert_eq!((t["title"].as_str(), t["task"].as_str()), (Some("A Bell pair"), Some("Entangle two qubits."))); + assert!(t["requirements"][0].as_str().unwrap().contains("00 with probability 0.5"), "{t:#}"); + } + #[test] fn an_exercise_outside_the_course_is_checked_as_a_lesson_is() { // A lesson's own exercise, given as a document: the same check, byte for byte, but for -- 2.54.0 (Apple Git-157) From 0d5f9b871af6d06541b0b4d8abee88c8b891bc8c Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 5 Nov 2026 17:04:49 -0500 Subject: [PATCH 2/5] wai-quantum-docs.yml: the course's tests are eleven, and this branch's is required by name The workflow pins how many course tests pass and names each one, since a module filter is satisfied by any one test. This branch adds any_exercise_is_shown_as_a_lesson_shows_its_own to engine::course, and the pin it now inherits from the branch below counts ten. So the step failed here with all eleven passing ("test result: ok. 11 passed" in the job's log, then "the course's tests did not all pass"). The count is eleven and the new name is the last in the list, the same lines the branch stacked above this one already holds, so the two merge as one change. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/wai-quantum-docs.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wai-quantum-docs.yml b/.github/workflows/wai-quantum-docs.yml index 4a8156e7c..1c5076594 100644 --- a/.github/workflows/wai-quantum-docs.yml +++ b/.github/workflows/wai-quantum-docs.yml @@ -125,7 +125,7 @@ jobs: run: | set -euo pipefail cargo test --locked --release --features full --lib -- engine::course:: 2>&1 | tee "$RUNNER_TEMP/course.log" - grep -qE '^test result: ok\. 10 passed; 0 failed' "$RUNNER_TEMP/course.log" || { echo "::error::the course's tests did not all pass"; exit 1; } + grep -qE '^test result: ok\. 11 passed; 0 failed' "$RUNNER_TEMP/course.log" || { echo "::error::the course's tests did not all pass"; exit 1; } for t in every_solution_passes_its_exercise_and_every_known_wrong_answer_fails_it \ a_check_says_what_went_wrong_from_the_engine_and_never_shows_the_solution \ no_hint_or_lesson_gives_the_solution \ @@ -135,7 +135,8 @@ jobs: a_check_is_the_same_bytes_every_time \ the_estimate_lesson_prices_the_program \ a_locale_tag_picks_the_course_s_locale \ - an_exercise_outside_the_course_is_checked_as_a_lesson_is; do + an_exercise_outside_the_course_is_checked_as_a_lesson_is \ + any_exercise_is_shown_as_a_lesson_shows_its_own; do grep -q "^test engine::course::tests::$t \.\.\. ok$" "$RUNNER_TEMP/course.log" || { echo "::error::$t did not run"; exit 1; } done - name: The command line's course and its languages -- 2.54.0 (Apple Git-157) From 64022d1b1864dc749ebc60f0962493e739797e0b Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Fri, 6 Nov 2026 03:00:47 -0500 Subject: [PATCH 3/5] wai-quantum-writer: one task digest names one check, the model loads exactly its manifest, and a reply's thinking is capped The loop, as reviewed: - A task is read in its canonical form (RFC 8785, what task_digest is taken over) and held to engine::course::exercise_form before the model is asked anything. The same task with its keys in another order, or a bound written 0.0, was two prompts and two checks under one digest; a job asking for a repair, a vacuous bound or an unknown constraint went to the model. - The transcript records check_memory_bytes, the memory each check's run is planned in, which decides what a check runs: the command line and the service used different caps and their transcripts did not say so. - The replay claim is scoped: the model's identity names the arithmetic it ran in (arch, os, and the BF16 matmul kernel: NEON fused multiply-add on aarch64, a scalar loop elsewhere), and a test holds the named kernel to the one that runs. The same files, task, options and engine give the same bytes in that arithmetic. The model wrapper: - CodeModel::open loads exactly the files wai-model.json lists, in its order, and refuses a manifest that lists none and a directory holding a model file it does not list or lacking one it lists (a stray shard sorting first was bound in place of the listed one). It checks sizes, and says so in the identity ("checked": "sizes"); open_verified (and --verify) reads every byte first and says "blake3". - answer_of drops <|START_TEXT|> and cuts at <|END_TEXT|>, which landed in the program. And the measured "no answer" (run 2: 23 of 43 attempts spent the whole budget thinking): a reply still thinking at its cap (Options::thinking_cap, two thirds of max_new unless --think-max says) has <|END_THINKING|><|START_TEXT|> fed for it and decodes its answer in the tokens left. The transcript records the cap and, per attempt, thinking_closed; `course` prints how many replies gave no program and how many were closed. The decode loop is a function over a small trait, so a test drives it with a model that thinks for ever. Each loop test failed on 63fb4a39c for the reason it states (a malformed task was written for; one task, two prompts), and the coder tests for theirs (the markers were in the answer; an unlisted shard was taken). The course's case names given as one string are shown by it. Co-Authored-By: Claude Opus 5.5 --- ...-quantum-writer-writes-program-for-task.md | 20 +- wai/wai-quantum-writer/Cargo.toml | 2 +- wai/wai-quantum-writer/src/coder.rs | 379 +++++++++++++++--- wai/wai-quantum-writer/src/lib.rs | 54 ++- wai/wai-quantum-writer/src/main.rs | 32 +- wai/wai-quantum-writer/tests/write.rs | 62 ++- wai/wai-quantum/src/engine/course.rs | 9 +- 7 files changed, 479 insertions(+), 79 deletions(-) diff --git a/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md b/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md index ddebe679f..422bd4da1 100644 --- a/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md +++ b/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md @@ -3,12 +3,24 @@ A new crate, `wai/wai-quantum-writer`, holds the loop the job service will write programs with. - **The task:** an exercise in the course's form (`wai.quantum.exercise/1`), shown to the model through `engine::course::task_of`. `task_of` and `engine::course::exercise` are new; `lesson` shows the same fields through it. + The loop reads a task in its canonical form (RFC 8785, the form its digest is taken over), so one `task_digest` names + one prompt and one check, and holds it to the course's form (`engine::course::exercise_form`) before the model is + asked anything. - **The loop:** the model writes a program, the engine checks it as it checks a lesson's answer, and what went wrong goes back for the next round, until a program passes or the rounds run out. - **The model:** North Mini Code 1.0 (Apache-2.0) on the pure-Rust runtime (`jouleclaw-loader-gguf`). Decoding is greedy, and the conversation is held in the model's cache, so a repair round reads only what is new. The loop takes - only the answer that follows the model's thinking. + only the answer that follows the model's thinking, without the markers around it. A reply still thinking at its cap + (two thirds of its tokens unless `--think-max` says otherwise) has its thinking closed for it and answers in the + tokens left, so a model that would think through its whole budget still writes a program. +- **Its files:** the model loads exactly the files its manifest (`wai.quantum.model/1`) lists, and refuses a manifest + that lists none and a directory with a model file it does not list or without one it lists. Opening checks their + sizes, and `--verify` reads every byte first; the model's identity says which (`"checked": "sizes"` or `"blake3"`). - **The transcript** (`wai.quantum.write/1`) names the task, the model and the prompt by digest (the model by the - digest of its file manifest, `wai.quantum.model/1`), and each attempt by its program's digest and the engine's check. - The same model files, task and engine give the same bytes. -- **The command line:** `wai-quantum-writer write`, `course`, `manifest` and `verify`. + digest of its file manifest, with how far its files were checked), the decoding with its thinking cap, the memory + each check's run was planned in (`check_memory_bytes`, which decides what a check runs), and each attempt by its + program's digest, the engine's check, and whether its thinking was closed. The same model files, task, options and + engine give the same bytes in the same arithmetic, which the model's identity names (architecture, operating system + and matmul kernel): on another, the logits can differ in their low bits and a near tie go the other way. +- **The command line:** `wai-quantum-writer write`, `course` (which also counts the replies that gave no program and + those whose thinking was closed), `manifest` and `verify`. diff --git a/wai/wai-quantum-writer/Cargo.toml b/wai/wai-quantum-writer/Cargo.toml index d6371aad1..017e9521d 100644 --- a/wai/wai-quantum-writer/Cargo.toml +++ b/wai/wai-quantum-writer/Cargo.toml @@ -3,7 +3,7 @@ name = "wai-quantum-writer" version = "0.1.0" edition = "2024" license = "Apache-2.0" -description = "WAI Quantum writes a program for a task: a model on WAI's own pure-Rust inference runtime writes it, the engine checks it against what the task asks exactly as it checks a lesson's answer, and what went wrong goes back for another attempt. Decoding is greedy, so the same model files, task and engine give the same programs, and every attempt is in a transcript named by digests." +description = "WAI Quantum writes a program for a task: a model on WAI's own pure-Rust inference runtime writes it, the engine checks it against what the task asks exactly as it checks a lesson's answer, and what went wrong goes back for another attempt. Decoding is greedy, so the same model files, task, options and engine give the same programs in the same arithmetic, which the transcript names (architecture, operating system, matmul kernel), and every attempt is in a transcript named by digests." repository = "https://git.transaction.science/Transaction-Science/open-standards" publish = false diff --git a/wai/wai-quantum-writer/src/coder.rs b/wai/wai-quantum-writer/src/coder.rs index 34813053d..d934004d0 100644 --- a/wai/wai-quantum-writer/src/coder.rs +++ b/wai/wai-quantum-writer/src/coder.rs @@ -8,10 +8,24 @@ //! `<|END_THINKING|>`); the loop is given only the answer. //! //! The model is named by the manifest of its files, `wai-model.json` beside them ([`manifest`]): -//! each file's name, size and BLAKE3 digest. Its digest is the digest of that manifest, so a -//! transcript names exactly the weights that wrote its programs. [`CodeModel::open`] refuses a -//! directory whose files do not have the sizes the manifest states; [`verify`] reads every -//! byte. +//! each file's name, size and BLAKE3 digest. Its digest is the digest of that manifest. +//! [`CodeModel::open`] loads exactly the files the manifest lists, in its order, and refuses a +//! manifest that lists none, and a directory that holds a model file the manifest does not list, +//! lacks one it lists, or whose files do not have the sizes it states. That checks sizes, not +//! bytes, and the model's identity says so (`"checked": "sizes"`); [`CodeModel::open_verified`] +//! first reads every byte against the manifest ([`verify`]), and says `"checked": "blake3"`. +//! So a transcript names the weights that wrote its programs as far as its model says they were +//! checked. +//! +//! The identity also names the arithmetic the weights ran in: the target's architecture and +//! operating system, and the matmul kernel the runtime takes there ([`MATMUL`]). Greedy decoding +//! gives the same tokens again only in the same arithmetic; on another kernel or platform maths +//! library the logits differ in their low bits, and a near tie can go the other way. +//! +//! A reply's thinking is capped ([`crate::Options::thinking_cap`]): a reply still thinking when it +//! reaches the cap has its thinking closed for it (`<|END_THINKING|>`, then `<|START_TEXT|>` where +//! the tokenizer has it), and its answer is decoded in the tokens left, so a model that would think +//! through its whole budget still answers. use crate::{Model, Reply}; use jouleclaw_loader_gguf::bpe_tokenizer::BpeTokenizer; @@ -28,32 +42,46 @@ pub const MANIFEST_FILE: &str = "wai-model.json"; pub const NAME: &str = "North Mini Code 1.0"; pub const LICENCE: &str = "Apache-2.0"; +/// The BF16 matmul kernel the runtime's projections take on this target, which fixes the +/// arithmetic of every logit (`jouleclaw-loader-gguf`'s `matmul_bf16`): on aarch64, NEON with fused +/// multiply-adds into two accumulators; elsewhere, a scalar loop into one. A transcript names it +/// with the architecture and the operating system (whose maths library the attention's and the +/// router's exponentials come from): what a replay must match to give the same tokens. +pub const MATMUL: &str = if cfg!(target_arch = "aarch64") { "bf16, NEON fused multiply-add, two accumulators" } else { "bf16, scalar multiply then add, one accumulator" }; + const START_THINKING: &str = "<|START_THINKING|>"; const END_THINKING: &str = "<|END_THINKING|>"; +const START_TEXT: &str = "<|START_TEXT|>"; +const END_TEXT: &str = "<|END_TEXT|>"; fn turn(role: &str, text: &str) -> String { format!("<|START_OF_TURN_TOKEN|><|{role}_TOKEN|>{text}<|END_OF_TURN_TOKEN|>") } -/// The answer in a reply: the text after the thinking, the whole reply when the model did -/// not think, and None while the thinking has not ended. +/// The answer in a reply: the text after the thinking, or the whole reply when the model did +/// not think, without the markers that open and close the answer's text (`<|START_TEXT|>`, and +/// `<|END_TEXT|>` with all after it); None while the thinking has not ended. pub fn answer_of(reply: &str) -> Option<&str> { - match (reply.find(START_THINKING), reply.find(END_THINKING)) { - (_, Some(end)) => Some(&reply[end + END_THINKING.len()..]), - (Some(_), None) => None, - (None, None) => Some(reply), - } + let after = match (reply.find(START_THINKING), reply.find(END_THINKING)) { + (_, Some(end)) => &reply[end + END_THINKING.len()..], + (Some(_), None) => return None, + (None, None) => reply, + }; + let after = after.find(START_TEXT).map_or(after, |at| &after[at + START_TEXT.len()..]); + Some(after.find(END_TEXT).map_or(after, |at| &after[..at])) } -/// The files that make up the model, in the order the manifest lists them. +/// Whether a name is a model file: what the manifest lists and the model reads. +fn is_model_file(n: &str) -> bool { + n == "config.json" || n == "tokenizer.json" || (n.starts_with("model") && n.ends_with(".safetensors")) +} + +/// The model files in `dir`, sorted by name: the order [`manifest`] lists them in. fn model_files(dir: &Path) -> Result, String> { let mut files: Vec = std::fs::read_dir(dir) .map_err(|e| format!("{}: {e}", dir.display()))? .filter_map(|e| e.ok().map(|e| e.path())) - .filter(|p| { - let n = p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - n == "config.json" || n == "tokenizer.json" || (n.starts_with("model") && n.ends_with(".safetensors")) - }) + .filter(|p| p.file_name().is_some_and(|n| is_model_file(&n.to_string_lossy()))) .collect(); files.sort(); if !files.iter().any(|p| p.ends_with("config.json")) || !files.iter().any(|p| p.ends_with("tokenizer.json")) { @@ -88,33 +116,95 @@ pub fn manifest(dir: &Path) -> Result { Ok(json!({ "schema": MANIFEST_SCHEMA, "name": NAME, "licence": LICENCE, "files": files })) } -/// Read the manifest in `dir` and check that each file it lists is there with its size. -fn read_manifest(dir: &Path) -> Result { +/// The manifest in `dir`, and the files it lists, in its order, each checked to be there with +/// the size it states. The manifest lists at least `config.json`, `tokenizer.json` and one +/// `model*.safetensors`, each once, by its plain name, and the directory holds no model file it +/// does not list: what the model loads is exactly what the manifest names. +pub fn read_manifest(dir: &Path) -> Result<(Value, Vec), String> { let path = dir.join(MANIFEST_FILE); let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e} (write it with `wai-quantum-writer manifest {}`)", path.display(), dir.display()))?; let m: Value = serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?; if m["schema"] != MANIFEST_SCHEMA { return Err(format!("{}: not a {MANIFEST_SCHEMA} manifest", path.display())); } - for f in m["files"].as_array().into_iter().flatten() { - let name = f["name"].as_str().unwrap_or_default(); - let size = std::fs::metadata(dir.join(name)).map(|m| m.len()).map_err(|e| format!("{name}: {e}"))?; + let listed = m["files"].as_array().filter(|f| !f.is_empty()).ok_or_else(|| format!("{}: it lists no files, so it names no weights", path.display()))?; + let mut files: Vec = Vec::new(); + for f in listed { + let name = f["name"].as_str().filter(|n| is_model_file(n) && !n.contains(['/', '\\'])).ok_or_else(|| format!("{}: {} is not a model file's plain name", path.display(), f["name"]))?; + if f["blake3"].as_str().is_none_or(|d| d.len() != 64 || !d.bytes().all(|b| b.is_ascii_hexdigit())) { + return Err(format!("{}: {name} has no BLAKE3 digest", path.display())); + } + let p = dir.join(name); + if files.contains(&p) { + return Err(format!("{}: it lists {name} twice", path.display())); + } + let size = std::fs::metadata(&p).map(|m| m.len()).map_err(|e| format!("{name}: the manifest lists it: {e}"))?; if Some(size) != f["size"].as_u64() { return Err(format!("{name}: {size} bytes, the manifest says {}", f["size"])); } + files.push(p); } - Ok(m) + for need in ["config.json", "tokenizer.json"] { + if !files.iter().any(|p| p.ends_with(need)) { + return Err(format!("{}: it does not list {need}", path.display())); + } + } + if !files.iter().any(|p| p.extension().is_some_and(|e| e == "safetensors")) { + return Err(format!("{}: it lists no weights (model*.safetensors)", path.display())); + } + if let Some(extra) = model_files(dir)?.into_iter().find(|p| !files.contains(p)) { + return Err(format!("{}: a model file the manifest does not list; move it out, or write the manifest again", extra.display())); + } + Ok((m, files)) } -/// Read every byte of the model in `dir` and compare it with its manifest. +/// Read every byte of the model in `dir` and compare it with its manifest: each file it lists, +/// and nothing beside them ([`read_manifest`]). pub fn verify(dir: &Path) -> Result<(), String> { - let stated = read_manifest(dir)?; - let now = manifest(dir)?; - if stated["files"] == now["files"] { - Ok(()) - } else { - Err(format!("{}: the files do not match their manifest", dir.display())) + let (stated, files) = read_manifest(dir)?; + for (f, p) in stated["files"].as_array().into_iter().flatten().zip(&files) { + if file_digest(p)? != f["blake3"].as_str().unwrap_or_default() { + return Err(format!("{}: its bytes do not match its digest in the manifest", p.display())); + } } + Ok(()) +} + +/// What one reply's decoding needs of a model: its greedy choice of the next token, feeding it +/// tokens, and its tokenizer. +trait Decoder { + /// The next token: the highest logit, the lowest token id on a tie. + fn next(&self) -> u32; + fn feed(&mut self, ids: &[u32]) -> Result<(), String>; + fn text(&self, ids: &[u32]) -> String; +} + +/// One reply, decoded greedily: its tokens; the last one when the reply stopped on it unfed, +/// because the answer so far was enough; and whether its thinking was closed for it. The reply +/// ends at a stop token, at `max_new` tokens, or when `done` says the answer is enough. A reply +/// still in its thinking at `think_max` tokens has `close` fed and kept as its own, and decodes +/// its answer in the tokens left. +fn decode_reply(d: &mut dyn Decoder, stops: &[u32], close: &[u32], max_new: usize, think_max: usize, done: &dyn Fn(&str) -> bool) -> Result<(Vec, Option, bool), String> { + let mut ids: Vec = Vec::new(); + let mut closed = false; + while ids.len() < max_new { + if !closed && ids.len() >= think_max && answer_of(&d.text(&ids)).is_none() { + closed = true; + d.feed(close)?; + ids.extend_from_slice(close); + continue; + } + let next = d.next(); + if stops.contains(&next) { + break; + } + ids.push(next); + if ids.len().is_multiple_of(4) && answer_of(&d.text(&ids)).is_some_and(done) { + return Ok((ids, Some(next), closed)); + } + d.feed(&[next])?; + } + Ok((ids, None, closed)) } /// North Mini Code 1.0, bound to its checkpoint, holding one conversation. @@ -124,29 +214,76 @@ pub struct CodeModel { identity: Value, bos: u32, stops: Vec, + /// What closes a thinking that reached its cap. + close: Vec, state: Option, logits: Vec, /// A token chosen and kept in the reply but not yet fed (the reply stopped on it). pending: Option, } +/// [`Decoder`] over the model's cache and the logits it left. +struct Live<'a> { + model: &'a Cohere2MoeModel, + tok: &'a BpeTokenizer, + state: &'a mut DecodeState, + logits: &'a mut Vec, +} + +impl Decoder for Live<'_> { + fn next(&self) -> u32 { + self.logits.iter().enumerate().fold((0usize, f32::NEG_INFINITY), |(bi, bv), (i, &v)| if v > bv { (i, v) } else { (bi, bv) }).0 as u32 + } + fn feed(&mut self, ids: &[u32]) -> Result<(), String> { + for &id in ids { + *self.logits = self.model.forward_token(id, self.state); + } + Ok(()) + } + fn text(&self, ids: &[u32]) -> String { + self.tok.decode(ids) + } +} + impl CodeModel { - /// Bind the checkpoint in `dir` (memory-mapped, nothing copied) after checking it - /// against its manifest's sizes. + /// Bind the checkpoint in `dir` (memory-mapped, nothing copied): exactly the files its + /// manifest lists, checked for their sizes ([`read_manifest`]). Its identity says so. pub fn open(dir: &Path) -> Result { - let m = read_manifest(dir)?; + Self::bind(dir, "sizes") + } + + /// [`CodeModel::open`], once every byte of the model has been read against its manifest + /// ([`verify`]). Its identity says so. + pub fn open_verified(dir: &Path) -> Result { + verify(dir)?; + Self::bind(dir, "blake3") + } + + fn bind(dir: &Path, checked: &str) -> Result { + let (m, files) = read_manifest(dir)?; let config: Cohere2MoeConfig = serde_json::from_slice(&std::fs::read(dir.join("config.json")).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?; - let shards = model_files(dir)? - .into_iter() + let shards = files + .iter() .filter(|p| p.extension().is_some_and(|e| e == "safetensors")) - .map(|p| read_safetensors_file(&p).map_err(|e| format!("{}: {e:?}", p.display()))) + .map(|p| read_safetensors_file(p).map_err(|e| format!("{}: {e:?}", p.display()))) .collect::, String>>()?; let model = Cohere2MoeModel::bind_sharded(config, shards).map_err(|e| format!("{e:?}"))?; let tok = BpeTokenizer::from_file(dir.join("tokenizer.json")).map_err(|e| format!("{e:?}"))?; let stops = ["<|END_OF_TURN_TOKEN|>", ""].iter().filter_map(|s| tok.special_id(s)).collect(); let bos = tok.special_id("").ok_or("the tokenizer has no ")?; - let identity = json!({ "name": m["name"], "licence": m["licence"], "digest": crate::digest(&m), "runtime": "jouleclaw-loader-gguf cohere2_moe" }); - Ok(CodeModel { model, tok, identity, bos, stops, state: None, logits: Vec::new(), pending: None }) + let close = [END_THINKING, START_TEXT].iter().filter_map(|s| tok.special_id(s)).collect::>(); + let close = if close.is_empty() { tok.encode(END_THINKING) } else { close }; + let identity = json!({ + "name": m["name"], + "licence": m["licence"], + "digest": crate::digest(&m), + "checked": checked, + "runtime": "jouleclaw-loader-gguf cohere2_moe", + "arch": std::env::consts::ARCH, + "os": std::env::consts::OS, + "matmul": MATMUL, + }); + Ok(CodeModel { model, tok, identity, bos, stops, close, state: None, logits: Vec::new(), pending: None }) } fn feed(&mut self, ids: &[u32]) -> Result<(), String> { @@ -173,28 +310,15 @@ impl Model for CodeModel { Ok(()) } - fn reply(&mut self, max_new: usize, done: &dyn Fn(&str) -> bool) -> Result { - let mut ids: Vec = Vec::new(); - for i in 0..max_new { - // Greedy: the highest logit, the lowest token id on a tie. - let next = self.logits.iter().enumerate().fold((0usize, f32::NEG_INFINITY), |(bi, bv), (i, &v)| if v > bv { (i, v) } else { (bi, bv) }).0 as u32; - if self.stops.contains(&next) { - break; - } - ids.push(next); - if i % 4 == 3 { - let so_far = self.tok.decode(&ids); - if answer_of(&so_far).is_some_and(done) { - self.pending = Some(next); - break; - } - } - self.feed(&[next])?; - } + fn reply(&mut self, max_new: usize, think_max: usize, done: &dyn Fn(&str) -> bool) -> Result { + let state = self.state.as_mut().ok_or("no conversation is open")?; + let mut live = Live { model: &self.model, tok: &self.tok, state, logits: &mut self.logits }; + let (ids, pending, closed) = decode_reply(&mut live, &self.stops, &self.close, max_new, think_max, done)?; + self.pending = pending; let text = self.tok.decode(&ids); let answer = answer_of(&text).map(str::to_string); let answer_tokens = answer.as_deref().map(|a| self.tok.encode(a).len()).unwrap_or(0).min(ids.len()); - Ok(Reply { answer, tokens: ids.len(), thinking_tokens: ids.len() - answer_tokens }) + Ok(Reply { answer, tokens: ids.len(), thinking_tokens: ids.len() - answer_tokens, thinking_closed: closed }) } fn answer(&mut self, user: &str) -> Result<(), String> { @@ -203,3 +327,146 @@ impl Model for CodeModel { self.feed(&ids) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{block_closed, program_of}; + + const STOP: u32 = 1_000_000; + const MARKS: [(u32, &str); 3] = [(1_000_001, START_THINKING), (1_000_002, END_THINKING), (1_000_003, START_TEXT)]; + + /// A model whose tokens are characters, each marker one token: it thinks ("hmm ") for as + /// long as it is let, and once its thinking is closed it answers with `answer`. + struct Thinker { + answer: Vec, + thought: usize, + answered: Option, + } + + impl Decoder for Thinker { + fn next(&self) -> u32 { + match self.answered { + Some(n) => self.answer.get(n).copied().unwrap_or(STOP), + None if self.thought == 0 => MARKS[0].0, + None => "hmm ".chars().nth(self.thought % 4).unwrap() as u32, + } + } + fn feed(&mut self, ids: &[u32]) -> Result<(), String> { + for &id in ids { + match (&mut self.answered, id) { + (None, 1_000_002) => self.answered = Some(0), + (None, _) => self.thought += 1, + (Some(_), 1_000_003) => {} + (Some(n), _) => *n += 1, + } + } + Ok(()) + } + fn text(&self, ids: &[u32]) -> String { + ids.iter().map(|&i| MARKS.iter().find(|(m, _)| *m == i).map_or_else(|| char::from_u32(i).unwrap().to_string(), |(_, s)| s.to_string())).collect() + } + } + + fn thinker() -> Thinker { + Thinker { answer: "```qasm\nx q;\n```".chars().map(|c| c as u32).collect(), thought: 0, answered: None } + } + + /// A reply still thinking at its cap has its thinking closed for it and answers in the + /// tokens left; without the cap the same model thinks through its whole budget and gives + /// no program. + #[test] + fn a_reply_still_thinking_at_its_cap_is_closed_and_answers() { + let close = [MARKS[1].0, MARKS[2].0]; + let mut d = thinker(); + let (ids, _, closed) = decode_reply(&mut d, &[STOP], &close, 200, 120, &block_closed).unwrap(); + let text = d.text(&ids); + assert!(closed && text.contains(END_THINKING), "{text}"); + assert_eq!(answer_of(&text).map(program_of).as_deref(), Some("x q;\n"), "{text}"); + assert!(ids.len() < 200, "the answer took the rest of the budget: {}", ids.len()); + let mut d = thinker(); + let (ids, pending, closed) = decode_reply(&mut d, &[STOP], &close, 200, 200, &block_closed).unwrap(); + assert!(!closed && pending.is_none() && ids.len() == 200 && answer_of(&d.text(&ids)).is_none()); + // A model that answers without thinking is never cut. + let mut d = Thinker { thought: 1, answered: Some(0), ..thinker() }; + let (ids, _, closed) = decode_reply(&mut d, &[STOP], &close, 200, 0, &block_closed).unwrap(); + assert!(!closed && answer_of(&d.text(&ids)).map(program_of).as_deref() == Some("x q;\n")); + } + + /// The answer is the text between the markers that open and close it: neither, nor any + /// control token after the answer, is part of the program. + #[test] + fn the_answer_is_the_text_between_its_markers() { + let reply = "<|START_THINKING|>a qubit<|END_THINKING|><|START_TEXT|>OPENQASM 3.0;\nqubit q;\nx q;<|END_TEXT|><|END_OF_TURN_TOKEN|>"; + assert_eq!(answer_of(reply), Some("OPENQASM 3.0;\nqubit q;\nx q;")); + assert_eq!(program_of(answer_of(reply).unwrap()), "OPENQASM 3.0;\nqubit q;\nx q;\n"); + let unclosed = "<|START_THINKING|>a<|END_THINKING|><|START_TEXT|>```qasm\nOPENQASM 3.0;\nqubit q;\n<|END_TEXT|>"; + assert!(!program_of(answer_of(unclosed).unwrap()).contains("<|"), "{}", program_of(answer_of(unclosed).unwrap())); + assert_eq!(answer_of("<|START_THINKING|>still"), None); + assert_eq!(answer_of("OPENQASM 3.0;"), Some("OPENQASM 3.0;")); + } + + fn model_dir(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("wai-qw-model-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + for (n, b) in [("config.json", "{}"), ("tokenizer.json", "{}"), ("model-00001-of-00001.safetensors", "weights")] { + std::fs::write(d.join(n), b).unwrap(); + } + let m = manifest(&d).unwrap(); + std::fs::write(d.join(MANIFEST_FILE), m.to_string()).unwrap(); + d + } + + /// The model loads exactly what its manifest lists: a directory with a model file the + /// manifest does not list, or without one it lists, and a manifest that lists none, are + /// refused; sizes are what opening checks, and only verify reads the bytes. + #[test] + fn a_model_directory_must_hold_exactly_its_manifest_s_files() { + let d = model_dir("exact"); + assert!(read_manifest(&d).is_ok() && verify(&d).is_ok()); + // A shard from another download, which would sort first and be bound in place of the listed one. + std::fs::write(d.join("model-0.safetensors"), "other").unwrap(); + assert!(read_manifest(&d).is_err(), "a model file the manifest does not list was taken"); + std::fs::remove_file(d.join("model-0.safetensors")).unwrap(); + // A listed file gone. + std::fs::rename(d.join("model-00001-of-00001.safetensors"), d.join("elsewhere")).unwrap(); + assert!(read_manifest(&d).is_err()); + std::fs::rename(d.join("elsewhere"), d.join("model-00001-of-00001.safetensors")).unwrap(); + // The same size, other bytes: opening checks sizes, so only verify sees it. + std::fs::write(d.join("model-00001-of-00001.safetensors"), "WEIGHTS").unwrap(); + assert!(read_manifest(&d).is_ok() && verify(&d).is_err()); + // A manifest that lists no files, or a path rather than a name. + let m: Value = serde_json::from_str(&std::fs::read_to_string(d.join(MANIFEST_FILE)).unwrap()).unwrap(); + for files in [json!([]), Value::Null, json!([{ "name": "../config.json", "size": 2, "blake3": "0".repeat(64) }])] { + let mut bad = m.clone(); + bad["files"] = files; + std::fs::write(d.join(MANIFEST_FILE), bad.to_string()).unwrap(); + assert!(read_manifest(&d).is_err(), "{bad}"); + } + let _ = std::fs::remove_dir_all(&d); + } + + /// The kernel the identity names is the one the runtime's projections take here: on + /// aarch64 its sums differ from the scalar loop's in their low bits, elsewhere they are the + /// scalar loop's. + #[test] + fn the_named_matmul_is_the_one_that_runs() { + use jouleclaw_loader_gguf::matmul_bf16::{matmul_bf16_neon, matmul_bf16_scalar}; + let (n, k) = (64, 512); + let mut x = 0x9e37_79b9_7f4a_7c15u64; + let mut rnd = || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 40) as f32 / (1u64 << 24) as f32 - 0.5 + }; + let a: Vec = (0..k).map(|_| rnd()).collect(); + let w: Vec = (0..n * k).flat_map(|_| ((rnd().to_bits() >> 16) as u16).to_le_bytes()).collect(); + let (mut s, mut v) = (vec![0f32; n], vec![0f32; n]); + matmul_bf16_scalar(&a, &w, &[], 1, n, k, &mut s); + matmul_bf16_neon(&a, &w, &[], 1, n, k, &mut v); + let differ = s.iter().zip(&v).any(|(x, y)| x.to_bits() != y.to_bits()); + assert_eq!(differ, MATMUL.contains("NEON"), "{MATMUL}"); + } +} diff --git a/wai/wai-quantum-writer/src/lib.rs b/wai/wai-quantum-writer/src/lib.rs index afaaf4ab9..1fff75897 100644 --- a/wai/wai-quantum-writer/src/lib.rs +++ b/wai/wai-quantum-writer/src/lib.rs @@ -8,10 +8,18 @@ //! the model in the engine's own words for another attempt. The loop stops at the first //! program that passes, or after its rounds. //! +//! The task is read in its canonical form (RFC 8785, the form its digest is taken over), so one +//! `task_digest` names one prompt and one check, whatever order or spelling of numbers the task +//! came in. It is held to the course's form before the model is asked anything +//! ([`course::exercise_form`]). +//! //! Decoding is greedy, so the same model files, task, options and engine give the same -//! programs, round for round: anyone who holds the model can replay a transcript and get -//! its bytes. The transcript ([`SCHEMA`]) names the model by the digest of its files, -//! the prompt by its version and digest, and each attempt by its program's digest and +//! programs, round for round, in the same arithmetic: the transcript names it (the model's +//! architecture, operating system and matmul kernel, `coder::MATMUL`), and a replay on the +//! same gives the transcript's bytes; on another, the logits can differ in their low bits and a +//! near tie go the other way. The transcript ([`SCHEMA`]) names the model by the digest of its +//! files and how far they were checked, the prompt by its version and digest, the decoding, +//! the memory each check's run was planned in, and each attempt by its program's digest and //! the engine's check. How long each step took is returned beside it ([`Written::timing`]), //! not inside it, since time is the one thing a replay does not reproduce. @@ -38,13 +46,26 @@ pub struct Options { pub rounds: usize, /// The most tokens one reply may take, its thinking included. pub max_new: usize, - /// The memory one check's run may take, in bytes. + /// The most tokens of a reply that may be thinking; none: two thirds of `max_new` + /// ([`Options::thinking_cap`]). + pub think_max: Option, + /// The memory one check's run may take, in bytes. It decides which programs a check + /// runs, and by which method, so the transcript records it. pub job_bytes: u64, } impl Default for Options { fn default() -> Options { - Options { rounds: 4, max_new: 3000, job_bytes: 1 << 30 } + Options { rounds: 4, max_new: 3000, think_max: None, job_bytes: 1 << 30 } + } +} + +impl Options { + /// The tokens a reply may think for before its thinking is closed for it and its answer + /// asked for in the tokens left: `think_max`, else two thirds of `max_new`, never more than + /// `max_new`. A model that thinks through its whole budget otherwise writes no program. + pub fn thinking_cap(&self) -> usize { + self.think_max.unwrap_or(self.max_new * 2 / 3).min(self.max_new) } } @@ -58,6 +79,9 @@ pub struct Reply { pub tokens: usize, /// The tokens of the reply that were thinking. pub thinking_tokens: usize, + /// Whether the reply reached the thinking cap still thinking, and had its thinking closed + /// for it. + pub thinking_closed: bool, } /// A model the loop asks: one conversation at a time, continued turn by turn, decoded @@ -70,8 +94,9 @@ pub trait Model { /// comes next. fn open(&mut self, system: &str, user: &str) -> Result<(), String>; /// The model's reply, until it ends its turn, `max_new` tokens have passed, or - /// `done` says the answer so far is enough. - fn reply(&mut self, max_new: usize, done: &dyn Fn(&str) -> bool) -> Result; + /// `done` says the answer so far is enough. A reply still thinking at `think_max` + /// tokens has its thinking closed, and answers in the tokens left. + fn reply(&mut self, max_new: usize, think_max: usize, done: &dyn Fn(&str) -> bool) -> Result; /// Continue the conversation with the next turn, after the model's last reply. fn answer(&mut self, user: &str) -> Result<(), String>; } @@ -89,6 +114,12 @@ pub fn digest(v: &Value) -> String { blake3::hash(&receipt::canonical(v)).to_hex().to_string() } +/// A task in its canonical form (RFC 8785), read back: the form [`digest`] is taken over, so +/// that what the model is shown and what the engine checks are what the digest names. +pub fn canonical_task(task: &Value) -> Result { + serde_json::from_slice(&receipt::canonical(task)).map_err(|e| format!("the task does not read back from its canonical form: {e}")) +} + /// The digest of a text: BLAKE3 over its UTF-8 bytes. pub fn text_digest(s: &str) -> String { blake3::hash(s.as_bytes()).to_hex().to_string() @@ -174,6 +205,9 @@ pub fn write(model: &mut dyn Model, task: &Value, opts: &Options) -> Result Result Result Result Result { if let Some(m) = flag(args, "--max-new") { o.max_new = m.parse().map_err(|_| "--max-new: a number")?; } + if let Some(t) = flag(args, "--think-max") { + o.think_max = Some(t.parse().map_err(|_| "--think-max: a number")?); + } Ok(o) } +/// The model in `dir`, its bytes read against its manifest first with `--verify`. +fn open(args: &[String], dir: &std::path::Path) -> Result { + if args.iter().any(|a| a == "--verify") { coder::CodeModel::open_verified(dir) } else { coder::CodeModel::open(dir) } +} + fn run(args: &[String]) -> Result<(), String> { match args.first().map(String::as_str) { Some("manifest") => { @@ -52,7 +66,7 @@ fn run(args: &[String]) -> Result<(), String> { let dir = PathBuf::from(flag(args, "--model").ok_or("write --model DIR TASK.json")?); let file = args.iter().skip(1).find(|a| a.ends_with(".json")).ok_or("write --model DIR TASK.json")?; let task: Value = serde_json::from_str(&std::fs::read_to_string(file).map_err(|e| format!("{file}: {e}"))?).map_err(|e| format!("{file}: {e}"))?; - let mut model = coder::CodeModel::open(&dir)?; + let mut model = open(args, &dir)?; let w = write(&mut model, &task, &options(args)?)?; println!("{}", serde_json::to_string_pretty(&w.transcript).unwrap()); for (i, (reply, check)) in w.timing.iter().enumerate() { @@ -66,12 +80,18 @@ fn run(args: &[String]) -> Result<(), String> { let only: Option> = flag(args, "--lessons").map(|s| s.split(',').map(str::to_string).collect()); let opts = options(args)?; std::fs::create_dir_all(&out).map_err(|e| e.to_string())?; - let mut model = coder::CodeModel::open(&dir)?; + let mut model = open(args, &dir)?; let mut results = std::fs::File::create(out.join("results.jsonl")).map_err(|e| e.to_string())?; let ids: Vec<&str> = course::lesson_ids()?.into_iter().filter(|id| only.as_ref().is_none_or(|o| o.iter().any(|x| x == id))).collect(); let mut passed_at: Vec> = Vec::new(); + let mut attempts = (0usize, 0usize, 0usize); for id in &ids { let w = write(&mut model, &course::exercise(id)?, &opts)?; + for a in w.transcript["attempts"].as_array().into_iter().flatten() { + attempts.0 += 1; + attempts.1 += usize::from(a["answered"] == false); + attempts.2 += usize::from(a["thinking_closed"] == true); + } let at = w.transcript["passed_round"].as_u64(); eprintln!("{id}: {}", at.map_or("never passed".to_string(), |r| format!("passed after {r} repair(s)"))); passed_at.push(at); @@ -85,6 +105,8 @@ fn run(args: &[String]) -> Result<(), String> { } let never: Vec<&&str> = ids.iter().zip(&passed_at).filter(|(_, p)| p.is_none()).map(|(l, _)| l).collect(); println!("never passed: {never:?}"); + let (tried, unanswered, closed) = attempts; + println!("replies: {tried}, of which {unanswered} gave no program and {closed} had their thinking closed at {} tokens", opts.thinking_cap()); Ok(()) } _ => Err("usage: wai-quantum-writer write --model DIR TASK.json | course --model DIR --out DIR | manifest DIR | verify DIR".into()), diff --git a/wai/wai-quantum-writer/tests/write.rs b/wai/wai-quantum-writer/tests/write.rs index e4388372c..d682fdf55 100644 --- a/wai/wai-quantum-writer/tests/write.rs +++ b/wai/wai-quantum-writer/tests/write.rs @@ -17,9 +17,9 @@ impl Model for Scripted { self.told = vec![system.to_string(), user.to_string()]; Ok(()) } - fn reply(&mut self, _max_new: usize, _done: &dyn Fn(&str) -> bool) -> Result { + fn reply(&mut self, _max_new: usize, _think_max: usize, _done: &dyn Fn(&str) -> bool) -> Result { let next = self.replies.remove(0); - Ok(Reply { answer: next.map(str::to_string), tokens: 10, thinking_tokens: 4 }) + Ok(Reply { answer: next.map(str::to_string), tokens: 10, thinking_tokens: 4, thinking_closed: false }) } fn answer(&mut self, user: &str) -> Result<(), String> { self.told.push(user.to_string()); @@ -108,3 +108,61 @@ fn the_program_is_the_answers_code_block() { assert!(block_closed("```qasm\nx\n```") && !block_closed("```qasm\nx\n")); assert!(prompt(&bell_task()).starts_with("Task: A Bell pair")); } + +/// A task is read in its canonical form, the form its digest names: the same task with its +/// keys in another order, or a whole number written as a float, is the same prompt, the same +/// check and the same transcript, and a bound written `0.0` is the bound 0. +#[test] +fn one_task_digest_names_one_prompt_and_one_check() { + let run = |task: Value| { + let mut m = Scripted { replies: vec![Some(BELL)], told: vec![] }; + let w = write(&mut m, &task, &Options { rounds: 0, ..Options::default() }).unwrap(); + (serde_json::to_vec(&w.transcript).unwrap(), m.told[1].clone(), w.transcript) + }; + let task = |d: Value, max: Value| json!({ "title": "A Bell pair", "intent": { "distribution": d }, "constraints": { "counts": { "cx": { "max": max } } } }); + let (a, prompt_a, t) = run(task(json!({ "00": 0.5, "11": 0.5 }), json!(0))); + let (b, prompt_b, _) = run(task(json!({ "11": 0.5, "00": 0.5 }), json!(0.0))); + assert_eq!(prompt_a, prompt_b, "one task, two prompts"); + assert_eq!(a, b, "one task, two transcripts"); + // The bound holds: the Bell pair has a cx, and the task allows none. + assert_eq!((t["passed"].as_bool(), t["attempts"][0]["check"]["constraints"][0]["holds"].as_bool()), (Some(false), Some(false)), "{t:#}"); +} + +/// A task is held to the course's form before the model is asked anything: a bound that is +/// not whole numbers, a constraint the course does not name, or a job that asks for more than +/// shots, seed and noise (a repair the transcript would not show) is refused. +#[test] +fn a_malformed_task_is_refused_before_the_model_is_asked() { + for extra in [ + json!({ "constraints": { "t_count": 0 } }), + json!({ "constraints": { "depth": { "max": 3 } } }), + json!({ "constraints": { "counts": { "cx": { "max": -1 } } } }), + json!({ "job": { "repair": true } }), + json!({ "job": { "dyadic_bits": 3 } }), + json!({ "job": 1 }), + json!({ "cases": vec![json!({ "inputs": {} }); 100] }), + ] { + let mut task = bell_task(); + for (k, v) in extra.as_object().unwrap() { + task[k] = v.clone(); + } + let mut m = Scripted { replies: vec![Some(BELL)], told: vec![] }; + let r = write(&mut m, &task, &Options::default()); + assert!(r.is_err(), "{task} was written for: {:#}", r.unwrap().transcript); + assert!(m.told.is_empty(), "{task}: the model was asked"); + } +} + +/// The transcript records what decides its checks and its replies beside the model: the +/// memory each check's run was planned in, the thinking cap, and each reply's thinking closed +/// or not. +#[test] +fn the_transcript_records_the_check_s_memory_and_the_thinking_cap() { + let mut m = Scripted { replies: vec![Some(BELL)], told: vec![] }; + let opts = Options { job_bytes: 512 << 20, max_new: 900, ..Options::default() }; + let t = write(&mut m, &bell_task(), &opts).unwrap().transcript; + assert_eq!(t["check_memory_bytes"].as_u64(), Some(512 << 20), "{t:#}"); + assert_eq!(t["decoding"], json!({ "method": "greedy", "max_new": 900, "think_max": 600, "rounds": 4 }), "{t:#}"); + assert_eq!(t["attempts"][0]["thinking_closed"], false, "{t:#}"); + assert_eq!(Options { think_max: Some(5000), ..Options::default() }.thinking_cap(), 3000); +} diff --git a/wai/wai-quantum/src/engine/course.rs b/wai/wai-quantum/src/engine/course.rs index fdbc33449..48a6ca525 100644 --- a/wai/wai-quantum/src/engine/course.rs +++ b/wai/wai-quantum/src/engine/course.rs @@ -665,8 +665,8 @@ fn cases_view(ex: &Value, loc: &str) -> Vec { .into_iter() .flatten() .map(|k| { - let mut v = json!({ "name": word(&k["name"], loc), "inputs": k["inputs"] }); - if let Some(i) = k.get("intent") { + let mut v = json!({ "name": text_in(&k["name"], loc), "inputs": k["inputs"] }); + if let Some(i) = k.get("intent").filter(|i| !i.is_null()) { v["intent"] = public_intents(i, loc); } v @@ -1311,6 +1311,11 @@ mod tests { let t = task_of(&mine, "en"); assert_eq!((t["title"].as_str(), t["task"].as_str()), (Some("A Bell pair"), Some("Entangle two qubits."))); assert!(t["requirements"][0].as_str().unwrap().contains("00 with probability 0.5"), "{t:#}"); + // A case named with one string is shown by it. + let cased = json!({ "intent": { "distribution": { "1": 1.0 } }, "cases": [{ "name": "a flip", "inputs": { "err": 1 }, "intent": null }] }); + let t = task_of(&cased, "en"); + assert_eq!((t["cases"][0]["name"].as_str(), t["cases"][0].get("intent")), (Some("a flip"), None), "{t:#}"); + assert!(t["requirements"].as_array().unwrap().iter().any(|r| r.as_str().unwrap().contains("a flip")), "{t:#}"); } #[test] -- 2.54.0 (Apple Git-157) From 8c3e3d09f0cad3a7a3bef91a3b891c7e7fef2f66 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Fri, 6 Nov 2026 03:09:57 -0500 Subject: [PATCH 4/5] wai-quantum-writer: a reply's thinking is counted on its own tokens thinking_tokens was the reply's tokens less its answer re-encoded, which is not the tokens the model gave (a re-encoding splits text its own way, and the answer's markers counted as thinking). It is now counted on the reply's tokens: up to and including the one that ends its thinking, all of a reply whose thinking never ended, none of one that never thought. What run 3 measures of the thinking cap rests on it. Co-Authored-By: Claude Opus 5.5 --- wai/wai-quantum-writer/src/coder.rs | 30 +++++++++++++++++++++++++++-- wai/wai-quantum-writer/src/lib.rs | 3 ++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/wai/wai-quantum-writer/src/coder.rs b/wai/wai-quantum-writer/src/coder.rs index d934004d0..06a17886a 100644 --- a/wai/wai-quantum-writer/src/coder.rs +++ b/wai/wai-quantum-writer/src/coder.rs @@ -170,6 +170,17 @@ pub fn verify(dir: &Path) -> Result<(), String> { Ok(()) } +/// The tokens of a reply that were thinking, counted on the tokens it gave: those up to and +/// including the one that ends its thinking; every one when its thinking did not end; none +/// when it did not think. The markers are the tokenizer's own tokens (`start`, `end`). +fn thinking_len(ids: &[u32], start: Option, end: Option) -> usize { + match (end.and_then(|e| ids.iter().position(|&i| i == e)), start.is_some_and(|s| ids.contains(&s))) { + (Some(at), _) => at + 1, + (None, true) => ids.len(), + (None, false) => 0, + } +} + /// What one reply's decoding needs of a model: its greedy choice of the next token, feeding it /// tokens, and its tokenizer. trait Decoder { @@ -317,8 +328,8 @@ impl Model for CodeModel { self.pending = pending; let text = self.tok.decode(&ids); let answer = answer_of(&text).map(str::to_string); - let answer_tokens = answer.as_deref().map(|a| self.tok.encode(a).len()).unwrap_or(0).min(ids.len()); - Ok(Reply { answer, tokens: ids.len(), thinking_tokens: ids.len() - answer_tokens, thinking_closed: closed }) + let thinking_tokens = thinking_len(&ids, self.tok.special_id(START_THINKING), self.tok.special_id(END_THINKING)); + Ok(Reply { answer, tokens: ids.len(), thinking_tokens, thinking_closed: closed }) } fn answer(&mut self, user: &str) -> Result<(), String> { @@ -393,6 +404,21 @@ mod tests { assert!(!closed && answer_of(&d.text(&ids)).map(program_of).as_deref() == Some("x q;\n")); } + /// A reply's thinking is counted on its own tokens: up to the one that ends it, all of a + /// reply that never ends it, none of one that never thought. + #[test] + fn a_reply_s_thinking_is_counted_on_its_tokens() { + let (s, e) = (Some(MARKS[0].0), Some(MARKS[1].0)); + let ids = |t: &[u32]| t.to_vec(); + assert_eq!(thinking_len(&ids(&[MARKS[0].0, 104, 109, MARKS[1].0, 120, 121]), s, e), 4); + assert_eq!(thinking_len(&ids(&[MARKS[0].0, 104, 109, 109]), s, e), 4); + assert_eq!(thinking_len(&ids(&[120, 121]), s, e), 0); + // Closed at its cap: the closing markers are the reply's, up to the end of its thinking. + let mut d = thinker(); + let (got, _, _) = decode_reply(&mut d, &[STOP], &[MARKS[1].0, MARKS[2].0], 200, 120, &block_closed).unwrap(); + assert_eq!(thinking_len(&got, s, e), 121); + } + /// The answer is the text between the markers that open and close it: neither, nor any /// control token after the answer, is part of the program. #[test] diff --git a/wai/wai-quantum-writer/src/lib.rs b/wai/wai-quantum-writer/src/lib.rs index 1fff75897..117a94eba 100644 --- a/wai/wai-quantum-writer/src/lib.rs +++ b/wai/wai-quantum-writer/src/lib.rs @@ -77,7 +77,8 @@ pub struct Reply { pub answer: Option, /// Every token of the reply, its thinking included. pub tokens: usize, - /// The tokens of the reply that were thinking. + /// The tokens of the reply that were thinking: up to and including the one that ended its + /// thinking, every one of a reply whose thinking did not end. pub thinking_tokens: usize, /// Whether the reply reached the thinking cap still thinking, and had its thinking closed /// for it. -- 2.54.0 (Apple Git-157) From 205234abc5c9c21445e945497ba1cf861bd4833f Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Fri, 6 Nov 2026 08:01:21 -0500 Subject: [PATCH 5/5] wai-quantum-writer: the thinking cap stays within max_new, a verified model is hashed through its mapping, and a transcript's options and maths library can be given again The second review's findings in the loop and the model wrapper: - The cap's edges. A close at the cap could take a reply past max_new (201 tokens of 200), and one with too few tokens left gave a program cut short to check. A close now needs room for itself and MIN_ANSWER (64) tokens; otherwise the reply ends with no answer. A reply that thinks again after the close is closed again, and answer_of reads the text after the last close. An empty answer is no program: the model is asked again, the engine not asked to check nothing. - "checked": "blake3" held only at check time: verify hashed files by name and bind mapped them by name again. open_verified now opens each file once, hashes it through the mapping the model reads (map_checked) and parses from that mapping; the config and tokenizer from the bytes hashed. The docs say what holds after mapping. - Replay. --check-memory-bytes gives the command line the check's memory a service planned in, so every option a transcript records can be given again. The identity names the maths library (glibc's version, or macOS's release and build), whose exponentials and sines shape the logits. The matmul test now runs the dispatcher the projections call (matmul_bf16_auto, one row and three, and the batch). Tests failing on fbbee6fd9 for the reasons they state: the_cap_keeps_within_max_new_and_closes_a_reopened_thinking (201 tokens of 200), every_recorded_option_can_be_given (the flag ignored), an_empty_answer_is_no_program (an empty program checked). New: a_shard_is_checked_through_the_mapping_the_model_reads, the_maths_library_is_named. Co-Authored-By: Claude Opus 5.5 --- ...-quantum-writer-writes-program-for-task.md | 12 +- wai/wai-quantum-writer/Cargo.lock | 1 + wai/wai-quantum-writer/Cargo.toml | 4 +- wai/wai-quantum-writer/src/coder.rs | 196 ++++++++++++++++-- wai/wai-quantum-writer/src/lib.rs | 7 +- wai/wai-quantum-writer/src/main.rs | 26 ++- wai/wai-quantum-writer/tests/write.rs | 11 + 7 files changed, 224 insertions(+), 33 deletions(-) diff --git a/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md b/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md index 422bd4da1..1741468f5 100644 --- a/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md +++ b/wai/changes/2026-11-05-quantum-writer-writes-program-for-task.md @@ -12,15 +12,19 @@ A new crate, `wai/wai-quantum-writer`, holds the loop the job service will write greedy, and the conversation is held in the model's cache, so a repair round reads only what is new. The loop takes only the answer that follows the model's thinking, without the markers around it. A reply still thinking at its cap (two thirds of its tokens unless `--think-max` says otherwise) has its thinking closed for it and answers in the - tokens left, so a model that would think through its whole budget still writes a program. + tokens left, so a model that would think through its whole budget still writes a program. The cap never takes a reply + past `max_new`: with fewer than 64 tokens left for an answer, the reply ends with no answer instead; a reply that thinks + again after the close is closed again; and an empty answer is no program. - **Its files:** the model loads exactly the files its manifest (`wai.quantum.model/1`) lists, and refuses a manifest that lists none and a directory with a model file it does not list or without one it lists. Opening checks their - sizes, and `--verify` reads every byte first; the model's identity says which (`"checked": "sizes"` or `"blake3"`). + sizes, and `--verify` hashes every byte through the mapping the model then reads, each file opened once; the model's + identity says which (`"checked": "sizes"` or `"blake3"`). - **The transcript** (`wai.quantum.write/1`) names the task, the model and the prompt by digest (the model by the digest of its file manifest, with how far its files were checked), the decoding with its thinking cap, the memory each check's run was planned in (`check_memory_bytes`, which decides what a check runs), and each attempt by its program's digest, the engine's check, and whether its thinking was closed. The same model files, task, options and - engine give the same bytes in the same arithmetic, which the model's identity names (architecture, operating system - and matmul kernel): on another, the logits can differ in their low bits and a near tie go the other way. + engine give the same bytes in the same arithmetic, which the model's identity names (architecture, operating system, + maths library and matmul kernel): on another, the logits can differ in their low bits and a near tie go the other way. + `--check-memory-bytes` gives the command line the check's memory a service planned in. - **The command line:** `wai-quantum-writer write`, `course` (which also counts the replies that gave no program and those whose thinking was closed), `manifest` and `verify`. diff --git a/wai/wai-quantum-writer/Cargo.lock b/wai/wai-quantum-writer/Cargo.lock index 682f671ce..3cb12b49c 100644 --- a/wai/wai-quantum-writer/Cargo.lock +++ b/wai/wai-quantum-writer/Cargo.lock @@ -611,6 +611,7 @@ version = "0.1.0" dependencies = [ "blake3", "jouleclaw-loader-gguf", + "memmap2", "serde_json", "wai-quantum", ] diff --git a/wai/wai-quantum-writer/Cargo.toml b/wai/wai-quantum-writer/Cargo.toml index 017e9521d..060306117 100644 --- a/wai/wai-quantum-writer/Cargo.toml +++ b/wai/wai-quantum-writer/Cargo.toml @@ -17,10 +17,12 @@ blake3 = "1" # The model the loop runs: North Mini Code 1.0 (Apache-2.0) on the pure-Rust # inference substrate (jouleclaw-loader-gguf), greedy decode from the checkpoint mmap. jouleclaw-loader-gguf = { path = "../../jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf", features = ["rayon"], optional = true } +# The model's shards mapped from one handle each, so open_verified hashes the bytes it maps. +memmap2 = { version = "0.9", optional = true } [features] default = ["coder"] -coder = ["dep:jouleclaw-loader-gguf"] +coder = ["dep:jouleclaw-loader-gguf", "dep:memmap2"] [[bin]] name = "wai-quantum-writer" diff --git a/wai/wai-quantum-writer/src/coder.rs b/wai/wai-quantum-writer/src/coder.rs index 06a17886a..ca626527b 100644 --- a/wai/wai-quantum-writer/src/coder.rs +++ b/wai/wai-quantum-writer/src/coder.rs @@ -13,12 +13,14 @@ //! manifest that lists none, and a directory that holds a model file the manifest does not list, //! lacks one it lists, or whose files do not have the sizes it states. That checks sizes, not //! bytes, and the model's identity says so (`"checked": "sizes"`); [`CodeModel::open_verified`] -//! first reads every byte against the manifest ([`verify`]), and says `"checked": "blake3"`. -//! So a transcript names the weights that wrote its programs as far as its model says they were -//! checked. +//! hashes every byte against the manifest as it maps it, each file opened once, and says +//! `"checked": "blake3"`: the bytes the model reads are the bytes hashed, as they were when +//! mapped. So a transcript names the weights that wrote its programs as far as its model says +//! they were checked. //! //! The identity also names the arithmetic the weights ran in: the target's architecture and -//! operating system, and the matmul kernel the runtime takes there ([`MATMUL`]). Greedy decoding +//! operating system, the platform's maths library ([`maths_library`]), and the matmul kernel the +//! runtime takes there ([`MATMUL`]). Greedy decoding //! gives the same tokens again only in the same arithmetic; on another kernel or platform maths //! library the logits differ in their low bits, and a near tie can go the other way. //! @@ -30,7 +32,9 @@ use crate::{Model, Reply}; use jouleclaw_loader_gguf::bpe_tokenizer::BpeTokenizer; use jouleclaw_loader_gguf::cohere2_moe::{Cohere2MoeConfig, Cohere2MoeModel, DecodeState}; -use jouleclaw_loader_gguf::safetensors_reader::read_safetensors_file; +use jouleclaw_loader_gguf::safetensors_reader::parse_safetensors_mmap; +use jouleclaw_loader_gguf::PrefetchPolicy; +use std::sync::Arc; use serde_json::{json, Value}; use std::io::Read; use std::path::{Path, PathBuf}; @@ -61,8 +65,11 @@ fn turn(role: &str, text: &str) -> String { /// The answer in a reply: the text after the thinking, or the whole reply when the model did /// not think, without the markers that open and close the answer's text (`<|START_TEXT|>`, and /// `<|END_TEXT|>` with all after it); None while the thinking has not ended. +/// A reply that thinks again after its thinking closed is thinking until it closes again, and +/// its answer is what follows its last close. pub fn answer_of(reply: &str) -> Option<&str> { - let after = match (reply.find(START_THINKING), reply.find(END_THINKING)) { + let after = match (reply.rfind(START_THINKING), reply.rfind(END_THINKING)) { + (Some(start), Some(end)) if start > end => return None, (_, Some(end)) => &reply[end + END_THINKING.len()..], (Some(_), None) => return None, (None, None) => reply, @@ -181,6 +188,52 @@ fn thinking_len(ids: &[u32], start: Option, end: Option) -> usize { } } +/// A shard mapped from one handle, and, given its digest, checked by hashing that mapping: the +/// bytes the model reads are the bytes hashed, never a file opened again by its name. A file +/// replaced after this (renamed over) is not seen, since the mapping holds the one opened. A file +/// written in place after it can change what the mapping reads, since a mapping reads the file +/// and is not a copy of it: what `"checked": "blake3"` holds to is the file as it was mapped. +fn map_checked(p: &Path, digest: Option<&str>) -> Result, String> { + let f = std::fs::File::open(p).map_err(|e| format!("{}: {e}", p.display()))?; + // SAFETY: a read-only mapping of a regular file, as the loader's own reader makes. + let mapped = Arc::new(unsafe { memmap2::Mmap::map(&f) }.map_err(|e| format!("{}: {e}", p.display()))?); + if let Some(want) = digest { + let mut h = blake3::Hasher::new(); + for chunk in mapped.chunks(1 << 24) { + h.update(chunk); + } + if h.finalize().to_hex().as_str() != want { + return Err(format!("{}: its bytes do not match its digest in the manifest", p.display())); + } + } + Ok(mapped) +} + +/// The platform's maths library, whose exponentials, powers and sines the attention, the router +/// and the rotary embedding take, and so part of the arithmetic a transcript names: glibc's +/// version on Linux, macOS's release and build (its libsystem_m) on macOS. +pub fn maths_library() -> String { + #[cfg(all(target_os = "linux", target_env = "gnu"))] + { + unsafe extern "C" { + fn gnu_get_libc_version() -> *const std::ffi::c_char; + } + // SAFETY: glibc returns a static NUL-terminated string. + let v = unsafe { std::ffi::CStr::from_ptr(gnu_get_libc_version()) }; + format!("glibc {}", v.to_string_lossy()) + } + #[cfg(target_os = "macos")] + { + let plist = std::fs::read_to_string("/System/Library/CoreServices/SystemVersion.plist").unwrap_or_default(); + let key = |k: &str| plist.split(&format!("{k}")).nth(1).and_then(|r| r.split("").nth(1)).and_then(|r| r.split("").next()).unwrap_or("?").to_string(); + format!("libsystem_m of macOS {} ({})", key("ProductVersion"), key("ProductBuildVersion")) + } + #[cfg(not(any(all(target_os = "linux", target_env = "gnu"), target_os = "macos")))] + { + format!("the {} {} C library, version not read", std::env::consts::OS, if cfg!(target_env = "musl") { "musl" } else { "platform" }) + } +} + /// What one reply's decoding needs of a model: its greedy choice of the next token, feeding it /// tokens, and its tokenizer. trait Decoder { @@ -190,16 +243,25 @@ trait Decoder { fn text(&self, ids: &[u32]) -> String; } +/// The fewest tokens a reply must have left, past the markers that close its thinking, for the +/// thinking to be closed for it at its cap: fewer, and the reply ends with no answer, never a +/// program cut short. +pub const MIN_ANSWER: usize = 64; + /// One reply, decoded greedily: its tokens; the last one when the reply stopped on it unfed, /// because the answer so far was enough; and whether its thinking was closed for it. The reply -/// ends at a stop token, at `max_new` tokens, or when `done` says the answer is enough. A reply -/// still in its thinking at `think_max` tokens has `close` fed and kept as its own, and decodes -/// its answer in the tokens left. +/// ends at a stop token, at `max_new` tokens (never more), or when `done` says the answer is +/// enough. A reply thinking at `think_max` tokens or after, its thinking opened again +/// included, has `close` fed and kept as its own and decodes its answer in the tokens left, +/// when they are [`MIN_ANSWER`] or more; otherwise it ends there, with no answer. fn decode_reply(d: &mut dyn Decoder, stops: &[u32], close: &[u32], max_new: usize, think_max: usize, done: &dyn Fn(&str) -> bool) -> Result<(Vec, Option, bool), String> { let mut ids: Vec = Vec::new(); let mut closed = false; while ids.len() < max_new { - if !closed && ids.len() >= think_max && answer_of(&d.text(&ids)).is_none() { + if ids.len() >= think_max && answer_of(&d.text(&ids)).is_none() { + if ids.len() + close.len() + MIN_ANSWER > max_new { + break; + } closed = true; d.feed(close)?; ids.extend_from_slice(close); @@ -266,20 +328,33 @@ impl CodeModel { /// [`CodeModel::open`], once every byte of the model has been read against its manifest /// ([`verify`]). Its identity says so. pub fn open_verified(dir: &Path) -> Result { - verify(dir)?; Self::bind(dir, "blake3") } fn bind(dir: &Path, checked: &str) -> Result { let (m, files) = read_manifest(dir)?; - let config: Cohere2MoeConfig = serde_json::from_slice(&std::fs::read(dir.join("config.json")).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?; + let verify = checked == "blake3"; + let digest_of = |p: &Path| m["files"].as_array().into_iter().flatten().find(|f| f["name"].as_str() == p.file_name().and_then(|n| n.to_str())).and_then(|f| f["blake3"].as_str()).map(str::to_string); + // Each file opened once: what is checked is what is read. + let small = |name: &str| -> Result, String> { + let p = dir.join(name); + let bytes = std::fs::read(&p).map_err(|e| format!("{}: {e}", p.display()))?; + if verify && Some(blake3::hash(&bytes).to_hex().to_string()) != digest_of(&p) { + return Err(format!("{}: its bytes do not match its digest in the manifest", p.display())); + } + Ok(bytes) + }; + let config: Cohere2MoeConfig = serde_json::from_slice(&small("config.json")?).map_err(|e| e.to_string())?; let shards = files .iter() .filter(|p| p.extension().is_some_and(|e| e == "safetensors")) - .map(|p| read_safetensors_file(p).map_err(|e| format!("{}: {e:?}", p.display()))) + .map(|p| { + let mapped = map_checked(p, if verify { digest_of(p) } else { None }.as_deref())?; + parse_safetensors_mmap(mapped, PrefetchPolicy::default()).map_err(|e| format!("{}: {e:?}", p.display())) + }) .collect::, String>>()?; let model = Cohere2MoeModel::bind_sharded(config, shards).map_err(|e| format!("{e:?}"))?; - let tok = BpeTokenizer::from_file(dir.join("tokenizer.json")).map_err(|e| format!("{e:?}"))?; + let tok = BpeTokenizer::from_json_bytes(&small("tokenizer.json")?).map_err(|e| format!("{e:?}"))?; let stops = ["<|END_OF_TURN_TOKEN|>", ""].iter().filter_map(|s| tok.special_id(s)).collect(); let bos = tok.special_id("").ok_or("the tokenizer has no ")?; let close = [END_THINKING, START_TEXT].iter().filter_map(|s| tok.special_id(s)).collect::>(); @@ -292,6 +367,7 @@ impl CodeModel { "runtime": "jouleclaw-loader-gguf cohere2_moe", "arch": std::env::consts::ARCH, "os": std::env::consts::OS, + "libm": maths_library(), "matmul": MATMUL, }); Ok(CodeModel { model, tok, identity, bos, stops, close, state: None, logits: Vec::new(), pending: None }) @@ -353,6 +429,8 @@ mod tests { answer: Vec, thought: usize, answered: Option, + /// Times it thinks again, from its start, when its thinking is closed. + reopens: usize, } impl Decoder for Thinker { @@ -366,9 +444,13 @@ mod tests { fn feed(&mut self, ids: &[u32]) -> Result<(), String> { for &id in ids { match (&mut self.answered, id) { + (_, 1_000_003) => {} + (None, 1_000_002) if self.reopens > 0 => { + self.reopens -= 1; + self.thought = 0; + } (None, 1_000_002) => self.answered = Some(0), (None, _) => self.thought += 1, - (Some(_), 1_000_003) => {} (Some(n), _) => *n += 1, } } @@ -380,7 +462,7 @@ mod tests { } fn thinker() -> Thinker { - Thinker { answer: "```qasm\nx q;\n```".chars().map(|c| c as u32).collect(), thought: 0, answered: None } + Thinker { answer: "```qasm\nx q;\n```".chars().map(|c| c as u32).collect(), thought: 0, answered: None, reopens: 0 } } /// A reply still thinking at its cap has its thinking closed for it and answers in the @@ -478,7 +560,8 @@ mod tests { /// scalar loop's. #[test] fn the_named_matmul_is_the_one_that_runs() { - use jouleclaw_loader_gguf::matmul_bf16::{matmul_bf16_neon, matmul_bf16_scalar}; + use jouleclaw_loader_gguf::fabric_dispatch::matmul_bf16_auto; + use jouleclaw_loader_gguf::matmul_bf16::{matmul_bf16_batch, matmul_bf16_scalar, BatchJob}; let (n, k) = (64, 512); let mut x = 0x9e37_79b9_7f4a_7c15u64; let mut rnd = || { @@ -487,12 +570,79 @@ mod tests { x ^= x << 17; (x >> 40) as f32 / (1u64 << 24) as f32 - 0.5 }; - let a: Vec = (0..k).map(|_| rnd()).collect(); + // Three rows: one as a decode step's projection takes it, all three as a prefill's. + let m = 3; + let a: Vec = (0..m * k).map(|_| rnd()).collect(); let w: Vec = (0..n * k).flat_map(|_| ((rnd().to_bits() >> 16) as u16).to_le_bytes()).collect(); - let (mut s, mut v) = (vec![0f32; n], vec![0f32; n]); - matmul_bf16_scalar(&a, &w, &[], 1, n, k, &mut s); - matmul_bf16_neon(&a, &w, &[], 1, n, k, &mut v); - let differ = s.iter().zip(&v).any(|(x, y)| x.to_bits() != y.to_bits()); - assert_eq!(differ, MATMUL.contains("NEON"), "{MATMUL}"); + let mut scalar = vec![0f32; m * n]; + matmul_bf16_scalar(&a, &w, &[], m, n, k, &mut scalar); + let differs = |got: &[f32], want: &[f32]| got.iter().zip(want).any(|(x, y)| x.to_bits() != y.to_bits()); + // What the model's projections call (cohere2_moe's proj and proj_m): the dispatcher. + let mut one = vec![0f32; n]; + matmul_bf16_auto(&a[..k], &w, &[], 1, n, k, &mut one); + let mut rows = vec![0f32; m * n]; + matmul_bf16_auto(&a, &w, &[], m, n, k, &mut rows); + // And what a decode step's expert window calls: the batch. + let mut batched = vec![0f32; n]; + matmul_bf16_batch(&mut [BatchJob { a: &a[..k], w: &w, n, k, out: &mut batched }, BatchJob { a: &a[k..2 * k], w: &w, n, k, out: &mut vec![0f32; n] }]); + let neon = MATMUL.contains("NEON"); + assert_eq!(differs(&one, &scalar[..n]), neon, "decode step: {MATMUL}"); + assert_eq!(differs(&rows, &scalar), neon, "prefill: {MATMUL}"); + assert_eq!(differs(&batched, &scalar[..n]), neon, "batch: {MATMUL}"); + // Whatever the kernel, the dispatcher's three paths agree with one another. + assert!(!differs(&one, &rows[..n]) && !differs(&batched, &one)); + } + + /// The thinking cap never takes a reply past `max_new`, and closes a thinking only where an + /// answer has room after it; a reply that thinks again after the close is closed again, and + /// its answer is what follows its last close. + #[test] + fn the_cap_keeps_within_max_new_and_closes_a_reopened_thinking() { + let close = [MARKS[1].0, MARKS[2].0]; + // The cap one token before the end: no room for the close and an answer. + let mut d = thinker(); + let (ids, _, closed) = decode_reply(&mut d, &[STOP], &close, 200, 199, &block_closed).unwrap(); + assert!(ids.len() <= 200, "the reply ran to {} tokens of 200", ids.len()); + assert!(!closed && answer_of(&d.text(&ids)).is_none(), "{}", d.text(&ids)); + // Room for the close, not for an answer: no answer, rather than a program cut short. + let mut d = thinker(); + let (ids, _, closed) = decode_reply(&mut d, &[STOP], &close, 200, 200 - close.len() - 10, &block_closed).unwrap(); + assert!(ids.len() <= 200 && !closed && answer_of(&d.text(&ids)).is_none(), "{}", d.text(&ids)); + // Thinking again after the close: closed again, and the answer is the last one. + let mut d = Thinker { reopens: 1, ..thinker() }; + let (ids, _, closed) = decode_reply(&mut d, &[STOP], &close, 400, 100, &block_closed).unwrap(); + let text = d.text(&ids); + assert!(closed && ids.len() <= 400, "{text}"); + assert_eq!(answer_of(&text).map(program_of).as_deref(), Some("x q;\n"), "{text}"); + assert_eq!(answer_of("<|START_THINKING|>a<|END_THINKING|>```qasm\nx q;\n```<|START_THINKING|>more"), None); + } + + /// A shard is mapped from the one handle its digest is checked through: replaced afterwards, + /// the mapping still reads the bytes that were checked; a mapping whose bytes are not the + /// digest's is refused. + #[test] + fn a_shard_is_checked_through_the_mapping_the_model_reads() { + let d = model_dir("map"); + let p = d.join("model-00001-of-00001.safetensors"); + let want = blake3::hash(b"weights").to_hex().to_string(); + let mapped = map_checked(&p, Some(&want)).unwrap(); + // Another file renamed over it after: the mapping is the one checked. + std::fs::write(d.join("other"), "WEIGHTS").unwrap(); + std::fs::rename(d.join("other"), &p).unwrap(); + assert_eq!(&mapped[..], b"weights"); + assert!(map_checked(&p, Some(&want)).unwrap_err().contains("do not match")); + assert!(map_checked(&p, None).is_ok()); + let _ = std::fs::remove_dir_all(&d); + } + + /// The maths library is named as the platform gives it. + #[test] + fn the_maths_library_is_named() { + let m = maths_library(); + if cfg!(all(target_os = "linux", target_env = "gnu")) { + assert!(m.starts_with("glibc 2."), "{m}"); + } else if cfg!(target_os = "macos") { + assert!(m.starts_with("libsystem_m of macOS ") && !m.contains('?'), "{m}"); + } } } diff --git a/wai/wai-quantum-writer/src/lib.rs b/wai/wai-quantum-writer/src/lib.rs index 117a94eba..dee623e97 100644 --- a/wai/wai-quantum-writer/src/lib.rs +++ b/wai/wai-quantum-writer/src/lib.rs @@ -15,7 +15,8 @@ //! //! Decoding is greedy, so the same model files, task, options and engine give the same //! programs, round for round, in the same arithmetic: the transcript names it (the model's -//! architecture, operating system and matmul kernel, `coder::MATMUL`), and a replay on the +//! architecture, operating system, maths library and matmul kernel, `coder::MATMUL`), and a +//! replay on the //! same gives the transcript's bytes; on another, the logits can differ in their low bits and a //! near tie go the other way. The transcript ([`SCHEMA`]) names the model by the digest of its //! files and how far they were checked, the prompt by its version and digest, the decoding, @@ -219,7 +220,9 @@ pub fn write(model: &mut dyn Model, task: &Value, opts: &Options) -> Result course::check_exercise(task, p, opts.job_bytes, "en")?, None => Value::Null, diff --git a/wai/wai-quantum-writer/src/main.rs b/wai/wai-quantum-writer/src/main.rs index 8557a665a..500bebb90 100644 --- a/wai/wai-quantum-writer/src/main.rs +++ b/wai/wai-quantum-writer/src/main.rs @@ -1,8 +1,8 @@ //! `wai-quantum-writer`: WAI Quantum writes a program for a task, on this machine. //! //! ```text -//! wai-quantum-writer write --model DIR TASK.json [--rounds N] [--max-new N] [--think-max N] [--verify] # the transcript, as JSON -//! wai-quantum-writer course --model DIR --out DIR [--rounds N] [--max-new N] [--think-max N] [--verify] [--lessons a,b] +//! wai-quantum-writer write --model DIR TASK.json [--rounds N] [--max-new N] [--think-max N] [--check-memory-bytes N] [--verify] # the transcript, as JSON +//! wai-quantum-writer course --model DIR --out DIR [--rounds N] [--max-new N] [--think-max N] [--check-memory-bytes N] [--verify] [--lessons a,b] //! wai-quantum-writer manifest DIR # write DIR/wai-model.json: every file's size and digest //! wai-quantum-writer verify DIR # read every byte of the model against its manifest //! ``` @@ -15,7 +15,9 @@ //! `--verify` reads every byte of the model against its manifest before it loads (the transcript's //! model then says `"checked": "blake3"`); without it only the files' sizes are checked, and it //! says `"sizes"`. `--think-max N` caps a reply's thinking at N tokens (two thirds of `--max-new` -//! when not given). +//! when not given). `--check-memory-bytes N` plans each check's run in N bytes (1 GiB when not +//! given), as a transcript's `check_memory_bytes` records it: to write a service's transcript +//! again, give the number it names (the service plans in one job's memory). use serde_json::{json, Value}; use std::io::Write; @@ -38,6 +40,9 @@ fn options(args: &[String]) -> Result { if let Some(t) = flag(args, "--think-max") { o.think_max = Some(t.parse().map_err(|_| "--think-max: a number")?); } + if let Some(b) = flag(args, "--check-memory-bytes") { + o.job_bytes = b.parse().map_err(|_| "--check-memory-bytes: a number of bytes")?; + } Ok(o) } @@ -120,3 +125,18 @@ fn main() { std::process::exit(1); } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Every option a transcript records can be given again, so the command line writes the + /// transcript a service wrote: the check's memory among them. + #[test] + fn every_recorded_option_can_be_given() { + let args: Vec = ["write", "--rounds", "2", "--max-new", "900", "--think-max", "500", "--check-memory-bytes", "77309411328"].iter().map(|s| s.to_string()).collect(); + let o = options(&args).unwrap(); + assert_eq!((o.rounds, o.max_new, o.thinking_cap(), o.job_bytes), (2, 900, 500, 72 << 30)); + assert!(options(&["--check-memory-bytes".to_string(), "lots".to_string()]).unwrap_err().contains("--check-memory-bytes")); + } +} diff --git a/wai/wai-quantum-writer/tests/write.rs b/wai/wai-quantum-writer/tests/write.rs index d682fdf55..e086ebc9e 100644 --- a/wai/wai-quantum-writer/tests/write.rs +++ b/wai/wai-quantum-writer/tests/write.rs @@ -166,3 +166,14 @@ fn the_transcript_records_the_check_s_memory_and_the_thinking_cap() { assert_eq!(t["attempts"][0]["thinking_closed"], false, "{t:#}"); assert_eq!(Options { think_max: Some(5000), ..Options::default() }.thinking_cap(), 3000); } + +/// An answer with nothing in it is no program: the model is asked again, and the engine is not +/// asked to check an empty one. +#[test] +fn an_empty_answer_is_no_program() { + let mut m = Scripted { replies: vec![Some(" \n "), Some(BELL)], told: vec![] }; + let t = write(&mut m, &bell_task(), &Options::default()).unwrap().transcript; + assert_eq!((t["attempts"][0]["answered"].as_bool(), t["attempts"][0]["check"].is_null()), (Some(false), true), "{t:#}"); + assert_eq!(m.told[2], NO_ANSWER); + assert_eq!(t["passed_round"].as_u64(), Some(1)); +} -- 2.54.0 (Apple Git-157)