open-standards/wai/wai-wasmq/mutations.py

#!/usr/bin/env python3
"""The checks of wasmQ, each removed in turn: the test that is of it must then fail.

A test that still passes with its check removed cannot fail for that check, and says
nothing of it. Each entry below names a check, where it is, how it is removed, and the
tests that must then fail. Run from the repository's root, on a build host (it builds
and tests a crate for each entry, so it takes a while):

    python3 wai/wai-wasmq/mutations.py            every entry
    python3 wai/wai-wasmq/mutations.py WORD ...   the entries whose name holds a WORD
    python3 wai/wai-wasmq/mutations.py --check    only that each statement is in the code once

Each file is restored after its entry, whatever happens. Exit 0 when every entry is
caught: its tests ran and at least one failed, or the test process died.
"""
import os
import subprocess
import sys

WASMQ = "wai/wai-wasmq"
QUANTUM = "wai/wai-quantum"
CLI = "wai/wai-quantum-cli"

# (name, the crate of the file, the file in it, the statement, what it becomes, the cargo
# test arguments, the tests, and optionally the crate whose tests they are). The tests are wasmQ's, but for a file of wai-quantum tested by
# that crate's own tests (`--lib`, or a test of its features), and the command line's for a
# file of the command line's.
M = [
    # The envelope (§2.3).
    ("the payload's magic", WASMQ, "src/envelope.rs", "    if take(payload, &mut p, 4)? != MAGIC {", "    if take(payload, &mut p, 4)? != MAGIC && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]),
    ("the contract's canonical form", WASMQ, "src/envelope.rs", "    if canonical(&contract) != text {", "    if canonical(&contract) != text && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]),
    ("the frames end with the payload", WASMQ, "src/envelope.rs", "    if p != payload.len() {", "    if p != payload.len() && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]),
    ("the contract's members", WASMQ, "src/envelope.rs", "    if canonical(&self::contract(&bundle, &method)) != text {", "    if canonical(&self::contract(&bundle, &method)) != text && false {", ["--test", "vectors"], ["each_refusal_is_refused"]),
    ("the manifest is the contract's", WASMQ, "src/envelope.rs", "    if manifest_bytes != self::manifest(&bundle, h.reading.program.n_qubits)?.as_slice() {", "    if manifest_bytes != self::manifest(&bundle, h.reading.program.n_qubits)?.as_slice() && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]),
    ("a frame's digest", WASMQ, "src/envelope.rs", "    if got != want {", "    if got != want && false {", ["--test", "adversarial"], ["a_changed_module_or_program", "a_prior_whose_hash"]),
    ("an input a double holds", WASMQ, "src/envelope.rs", "v.unsigned_abs() > MAX_EXACT) {", "v.unsigned_abs() > MAX_EXACT && false) {", ["--test", "review"], ["an_input_a_double_cannot_hold"]),
    ("one callout for each extern", WASMQ, "src/envelope.rs", "    if b.callouts.len() != p.externs.len() || b.callouts.iter().zip(&p.externs).any(|(c, d)| c.name != d.name) {", "    if false {", ["--test", "review"], ["a_contract_names_one_callout"]),
    ("the memory a host reaches", WASMQ, "src/envelope.rs", "        if f.memory.is_none() || !f.memory_exported {", "        if f.memory.is_none() {", ["--test", "review"], ["a_bundle_whose_memory_the_host_cannot_reach"]),
    ("a module with priors exports cabi_realloc", WASMQ, "src/envelope.rs", '            reachable(f, &m.name, "the host passes a prior", true)?;', '            reachable(f, &m.name, "the host passes a prior", false)?;', ["--test", "review"], ["a_module_with_priors_exports"]),
    ("the Clifford gates", WASMQ, "src/envelope.rs", "    if let Some(op) = p.ops.iter().find(", "    if let Some(op) = p.ops.iter().take(0).find(", ["--test", "review"], ["a_program_outside_the_clifford_gates"]),
    # What a run asks of its host (§4.7, §4.8).
    ("an envelope's bytes", WASMQ, "src/envelope.rs", "    if bytes.len() > MAX_ENVELOPE_BYTES {", "    if bytes.len() > MAX_ENVELOPE_BYTES && false {", ["--test", "sweep"], ["an_envelope_past"]),
    ("an envelope's frames", WASMQ, "src/envelope.rs", "    if frames >= MAX_ENVELOPE_BYTES as u64 {", "    if frames >= MAX_ENVELOPE_BYTES as u64 && false {", ["--test", "sweep"], ["an_envelope_past"]),
    ("a program's text", WASMQ, "src/envelope.rs", "    if program.len() > MAX_PROGRAM_BYTES {", "    if program.len() > MAX_PROGRAM_BYTES && false {", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("the code among a bundle's modules", WASMQ, "src/envelope.rs", "    if code > MAX_CODE_BYTES {", "    if code > MAX_CODE_BYTES && false {", ["--test", "sweep"], ["the_code_among_a_bundle"]),
    ("the memory among a bundle's modules", WASMQ, "src/envelope.rs", "    if pages > profile::MAX_MEMORY_PAGES {", "    if pages > profile::MAX_MEMORY_PAGES && false {", ["--test", "sweep"], ["the_modules_of_a_bundle"]),
    ("a memory's maximum, not its start", WASMQ, "src/envelope.rs", ".map(|(_, maximum)| maximum).sum();", ".map(|(initial, _)| initial.min(1)).sum();", ["--test", "sweep"], ["the_modules_of_a_bundle"]),
    ("a run's results", WASMQ, "src/envelope.rs", "    if bits > MAX_RESULT_BITS {", "    if bits > MAX_RESULT_BITS && false {", ["--test", "sweep"], ["a_bundle_whose_result"]),
    ("a run's results, at the limit", WASMQ, "src/envelope.rs", "    if bits > MAX_RESULT_BITS {", "    if bits >= MAX_RESULT_BITS {", ["--test", "sweep"], ["a_bundle_whose_result"]),
    ("a run's work", WASMQ, "src/envelope.rs", "    if work > MAX_HOST_WORK {", "    if work > MAX_HOST_WORK && false {", ["--test", "sweep"], ["a_run_no_host", "the_work_limit"]),
    ("a run's work, at the limit", WASMQ, "src/envelope.rs", "    if work > MAX_HOST_WORK {", "    if work >= MAX_HOST_WORK {", ["--test", "sweep"], ["the_work_limit", "a_run_s_fuel_is_bounded"]),
    ("a gate's work", WASMQ, "src/envelope.rs", "            Step::Instr(Instr::Gate(_)) => r,", "            Step::Instr(Instr::Gate(_)) => 1,", ["--test", "sweep"], ["the_work_a_run_asks"]),
    ("a measurement's work", WASMQ, "src/envelope.rs", "            Step::Instr(Instr::Measure { .. } | Instr::Reset(_)) => MEASURE_WORK * n * r,", "            Step::Instr(Instr::Measure { .. } | Instr::Reset(_)) => n * r,", ["--test", "sweep"], ["the_work_a_run_asks", "a_run_no_host"]),
    ("a tableau's work", WASMQ, "src/envelope.rs", "    let mut shot = 2 * n * r;", "    let mut shot = 0;", ["--test", "sweep"], ["the_work_a_run_asks"]),
    ("the calls of a callout", WASMQ, "src/envelope.rs", "                CALL_WORK * (1 + lists + u128::from(post && crate::abi::uses_memory(decl)))", "                CALL_WORK", ["--test", "sweep"], ["a_callout_s_work"]),
    ("the bits of a callout", WASMQ, "src/envelope.rs", " + BIT_WORK * (input.len() + output.len()) as u128 + 1", " + 1", ["--test", "sweep"], ["a_callout_s_work", "the_work_a_run_asks"]),
    ("the calls that make an instance", WASMQ, "src/envelope.rs", "        let calls = 1 + 2 * m.priors.len() + ", "        let calls = 1 + m.priors.len() + ", ["--test", "sweep"], ["a_callout_s_work", "the_work_a_run_asks"]),
    ("an instance's memory", WASMQ, "src/envelope.rs", ".map_or(0, |(initial, _)| u128::from(initial)) * 8192;", ".map_or(0, |(initial, _)| u128::from(initial));", ["--test", "sweep"], ["the_work_a_run_asks", "memory_a_module_grows"]),
    ("one instance for a run of `job`", WASMQ, "src/envelope.rs", '    let instances = if b.run.state == "shot" { b.run.shots } else { 1 };', "    let instances = b.run.shots;", ["--test", "sweep"], ["the_work_a_run_asks", "a_run_no_host"]),
    ("a module's code in the work", WASMQ, "src/envelope.rs", "        code += CODE_WORK * m.bytes.len() as u128;", "", ["--test", "sweep"], ["the_work_a_run_asks"]),
    ("the fuel in the work", WASMQ, "src/envelope.rs", " + code + FUEL_WORK * u128::from(b.run.per_job)", " + code", ["--test", "sweep"], ["a_run_s_fuel_is_bounded"]),
    # The reader's bounds (§4.8), and its nesting.
    ("the reader's work", QUANTUM, "src/qasm/elab.rs", "        if self.reading > self.bounds.reading {\n            return self.fail(", "        if self.reading > self.bounds.reading && false {\n            return self.fail(", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("the steps of a statement", QUANTUM, "src/qasm/elab.rs", "        if self.steps.len().saturating_add(more) > self.bounds.steps {", "        if self.steps.len().saturating_add(more) > self.bounds.steps && false {", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("the steps of a reading", QUANTUM, "src/qasm/elab.rs", "    if steps > bounds.steps {", "    if steps > bounds.steps && false {", ["--test", "sweep"], ["the_steps_of_a_program"]),
    ("the bits of a program", QUANTUM, "src/qasm/elab.rs", "        if held > u64::from(self.bounds.bits) {", "        if held > u64::from(self.bounds.bits) && false {", ["--test", "sweep"], ["a_bundle_whose_result"]),
    ("the bits of the calls", QUANTUM, "src/qasm/elab.rs", "        if self.callout_bits > self.bounds.callout_bits {", "        if self.callout_bits > self.bounds.callout_bits && false {", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("an array element's bits", QUANTUM, "src/qasm/elab.rs", "                    Ty::Bit(n) => u64::from(n.unwrap_or(1)),", "                    Ty::Bit(_) => 1u64,", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("an array's elements", QUANTUM, "src/qasm/elab.rs", "                if elements > u64::from(self.bounds.bits) {", "                if elements > u64::from(self.bounds.bits) && false {", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("the qubits a program declares", QUANTUM, "src/qasm/elab.rs", "                if total > u64::from(self.bounds.qubits) {", "                if total > u64::from(self.bounds.qubits) && false {", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("the qubits a program names", QUANTUM, "src/qasm/elab.rs", "            if m >= MAX_QUBITS || m >= self.bounds.qubits {", "            if m >= MAX_QUBITS {", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("the nesting", QUANTUM, "src/qasm/parse.rs", "        if self.depth > super::MAX_NESTING {", "        if self.depth > super::MAX_NESTING && false {", ["--test", "reading"], ["a_program_past_the_bounds"]),
    ("no measurement added to a hybrid program", QUANTUM, "src/qasm/elab.rs", "        if measured.is_empty() && !self.callouts {", "        if measured.is_empty() {", ["--test", "vectors"], ["the_vectors_are"]),
    # Lowering and the lifecycle (§5).
    ("a callout's width", WASMQ, "src/abi.rs", "            if n < 64 && v >> n != 0 {", "            if n < 64 && v >> n != 0 && false {", ["--test", "adversarial"], ["a_trap_and_a_result_past_its_width"]),
    ("a signed argument, sign-extended", WASMQ, "src/abi.rs", "                    && v >> (w - 1) & 1 == 1", "                    && false", ["--test", "vectors"], ["the_vectors_are"]),
    ("a list result is 4-aligned", WASMQ, "src/abi.rs", "            if !ret.is_multiple_of(4) {", "            if false {", ["--test", "vectors"], ["the_vectors_are"]),
    ("cabi_post after a list result", WASMQ, "src/abi.rs", "            if has_post {", "            if false {", ["--test", "vectors"], ["the_vectors_are"]),
    # The engines (§4).
    ("a body ends", WASMQ, "src/profile.rs", "                ops.finish().map_err(malformed)?;", "", ["--test", "review"], ["a_body_that_does_not_end"]),
    ("a frame's operands", WASMQ, "src/profile.rs", "                    operands = operands.max(fv.operand_stack_height());", "", ["--test", "engines"], ["a_native_frame"]),
    ("a frame's operands, in the interpreter", WASMQ, "src/interp.rs", "                    operands = operands.max(fv.operand_stack_height());", "", ["--test", "engines"], ["a_native_frame"]),
    ("the stack before an indirect call", WASMQ, "src/interp.rs", "                    self.suspend(f.weight)?;\n                    let i = self.pop() as u32 as usize;", "                    let i = self.pop() as u32 as usize;", ["--test", "review"], ["an_indirect_call_where_the_stack_ends"]),
    ("the engine reaches only an exported memory", WASMQ, "src/interp.rs", "        self.m.memory_exported.then_some(&mut self.mem[..])", "        Some(&mut self.mem[..])", ["--test", "review"], ["every_engine_reaches_a_module_s_memory"]),
    ("an argument is of its parameter's type", WASMQ, "src/interp.rs", "if matches!(t, ValType::I32 | ValType::F32) { a as u32 as u64 } else { a }", "if matches!(t, ValType::I32 | ValType::F32) { a } else { a }", ["--test", "review"], ["an_argument_is_read_as_a_value"]),
    ("a NaN is made canonical by the interpreter", WASMQ, "src/interp.rs", "fn c64(x: f64) -> u64 {\n    if x.is_nan() { CANON64 } else { x.to_bits() }", "fn c64(x: f64) -> u64 {\n    if false { CANON64 } else { x.to_bits() }", ["--test", "fuzz"], ["a_changed_module_is_refused"]),
    ("a page grown, in the interpreter's fuel", WASMQ, "src/interp.rs", "                self.charge(delta * meter::PAGE_FUEL)?;", "                self.charge(delta)?;", ["--test", "sweep"], ["memory_a_module_grows"]),
    ("a page grown, in the prepared module's fuel", WASMQ, "src/meter.rs", "if matches!(op, Operator::MemoryGrow { .. }) { PAGE_FUEL } else { 1 }", "1", ["--test", "engines"], ["the_fuel_is_the_same"]),
    ("a run's fuel is charged at its start", WASMQ, "src/meter.rs", "                charge_const(&mut f, &g, cost);", "", ["--test", "fuzz"], ["a_changed_module_is_refused"]),
    ("the frame weight in the prepared module", WASMQ, "src/meter.rs", "                    stack_in(&mut f, &g, weight_here);", "                    stack_in(&mut f, &g, 1);", ["--test", "engines"], ["the_stack_ends_at_the_same_call"]),
    ("the function limit", WASMQ, "src/profile.rs", '            Payload::FunctionSection(r) => count(r.count(), "functions", MAX_ITEMS)?,', "            Payload::FunctionSection(_) => {}", ["--test", "review"], ["the_function_limit"]),
    # The tableau: a determined outcome, word by word.
    ("the sign of a determined outcome", QUANTUM, "src/quantum_tableau.rs", "                let plus = (px & pz & z & !x) | (px & !pz & z & x) | (!px & pz & x & !z);", "                let plus = (px & pz & z & !x) | (!px & pz & x & !z);", ["--features", "full", "--lib", "--", "quantum_tableau::tests::a_determined_measurement"], []),
    ("a determined outcome's carry", QUANTUM, "src/quantum_tableau.rs", "                if (x.count_ones() & 1) == 1 {\n                    cx = !cx;\n                }", "", ["--features", "full", "--lib", "--", "quantum_tableau::tests::a_determined_measurement"], []),
    # A job (§6.5).
    ("the host's unsealed status, noted", QUANTUM, "src/engine/wasmq.rs", '            "status" if value == body_status(r) => {}', '            "status" => {}', ["--test", "review"], ["a_job_whose_status_or_method"]),
    ("verify's read, bounded by size", CLI, "src/main.rs", "            if f.metadata().map_err(|e| format!(\"{}: {e}\", p.display()))?.len() > MAX_JOB_BYTES {", "            if false {", ["--test", "wasmq"], ["verify_refuses_a_file_past"]),
    ("the signer is the receipt's key", WASMQ, "src/run.rs", 'signer: seal["key"].as_str().map(str::to_string)', "signer: None", ["--test", "review"], ["a_job_names_its_signer"]),
    ("the signature is the engine's check of the seal", WASMQ, "src/run.rs", "        signature: !c.reasons.iter().any(|x| sealing.contains(x)),", "        signature: true,", ["--test", "adversarial"], ["a_receipt_changed_after_sealing"]),
    ("a wasmQ job sealed in the canonical form", QUANTUM, "src/engine/wasmq.rs", '    if r["receipt"]["canon"] != "rfc8785" {', "    if false {", ["--features", "full", "--test", "wasmq_verdict"], ["every_reader"]),
    ("the result's members", QUANTUM, "src/engine/wasmq.rs", "    members(r, \"the result\", &RESULT, &[\"fault\"], &mut out);", "", ["--test", "sweep"], ["a_job_whose_result"]),
    ("the contract's form", QUANTUM, "src/engine/wasmq.rs", "    contract_form(c, &mut out);", "", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]),
    ("the contract's method", QUANTUM, "src/engine/wasmq.rs", "    if run[\"method\"] != METHOD {", "    if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]),
    ("the contract's shots", QUANTUM, "src/engine/wasmq.rs", "    if !whole(&run[\"shots\"], 1, MAX_SHOTS) {", "    if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]),
    ("what a result repeats from its contract", QUANTUM, "src/engine/wasmq.rs", "        if !same(&r[name], &want) {", "        if !same(&r[name], &want) && false {", ["--test", "sweep"], ["a_job_whose_result"]),
    ("fuel used within the run's budget", QUANTUM, "src/engine/wasmq.rs", "    if count(&fuel[\"used\"]).is_none_or(|u| u > per_job) {", "    if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]),
    ("a call within the call's budget", QUANTUM, "src/engine/wasmq.rs", "        if max.is_none_or(|m| m > per_call) {", "        if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]),
    ("a callout's fuel within its calls", QUANTUM, "src/engine/wasmq.rs", "        if all.is_none_or(|a| n.zip(max).is_none_or(", "        if false && all.is_none_or(|a| n.zip(max).is_none_or(", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]),
    ("the instances' fuel", QUANTUM, "src/engine/wasmq.rs", "    if count(&fuel[\"instances\"]).is_none_or(|n| u128::from(n) > made.saturating_mul(stretches).saturating_mul(u128::from(per_instance))) {", "    if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]),
    ("a fault's fuel within its phase's budget", QUANTUM, "src/engine/wasmq.rs", "            if used.is_none_or(|n| n > budget) {", "            if false {", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_fault_outside_a_call_is_within_the_instance_s_budget"]),
    ("the fuel adds up", QUANTUM, "src/engine/wasmq.rs", "    if spent.is_none() || spent != count(&fuel[\"used\"]) {", "    if (spent.is_none() || spent != count(&fuel[\"used\"])) && false {", ["--test", "sweep"], ["a_job_whose_result"]),
    ("a fault after every shot", QUANTUM, "src/engine/wasmq.rs", "            if shots_run.zip(shots).is_none_or(|(ran, all)| ran >= all) {", "            if false {", ["--test", "sweep"], ["a_job_whose_result"]),
    ("the counts' sum", QUANTUM, "src/engine/wasmq.rs", "    if total.is_none() || total != shots_run {", "    if (total.is_none() || total != shots_run) && false {", ["--test", "sweep"], ["a_job_whose_result"]),
    ("a record's shots are the run's", WASMQ, "src/run.rs", "        if call.shot >= b.run.shots {", "        if call.shot >= b.run.shots && false {", ["--test", "sweep"], ["a_record_of_a_shot"]),
    ("a record's shots are in order", WASMQ, "src/run.rs", "        if call.shot < last {", "        if call.shot < last && false {", ["--test", "sweep"], ["a_record_of_a_shot"]),
    # The command line.
    ("a job read from its text", CLI, "src/main.rs", "            } else if as_json || (v.get(\"request\").is_some() && v.get(\"result\").is_some()) {\n                let c = job::check_text(&src);", "            } else if as_json || (v.get(\"request\").is_some() && v.get(\"result\").is_some()) {\n                let c = job::check(&v);", ["--test", "wasmq"], ["a_job_is_read_from_its_text_as_every_job_is"]),
    ("no history for a wasmQ job", CLI, "src/main.rs", "            if wasmq && hist.is_some() {", "            if false {", ["--test", "wasmq"], ["json_and_lines_give"]),
    # Round three: a number past 2^53 - 1 is no count; a stopped call is one of the calls
    # made; a budget is at most 2^39 when packed; wasmQ's reading offers no method it cannot
    # use; what is reported alone is a verdict; routing, lines and the not-judged report.
    ("a whole number past 2^53 - 1 is no count", QUANTUM, "src/engine/wasmq.rs", "v.as_u64().filter(|&n| n <= MAX_EXACT).or_else(", "v.as_u64().or_else(", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_number_is_read_by_its_value", "engine::wasmq::tests::a_budget_past_two_to_the_53"]),
    ("a stopped call the fuel counts", QUANTUM, "src/engine/wasmq.rs", "None => out.push(format!(\"a call of {name:?} that stopped, where the fuel counts no call of it\")),", "None => {}", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_call_s_fault_is_one_of_the_calls_made"]),
    ("a stopped call within its callout's largest", QUANTUM, "src/engine/wasmq.rs", "Some(m) if count(&m[\"max\"]).zip(used).is_none_or(|(max, u)| u > max) =>", "Some(m) if false =>", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_call_s_fault_is_one_of_the_calls_made"]),
    ("a budget at most 2^39 when packed", WASMQ, "src/envelope.rs", "        if v > MAX_BUDGET {", "        if v > MAX_EXACT {", ["--test", "review"], ["a_budget_past_what_a_run_may_be_given"]),
    ("a hybrid reading offers no other method", QUANTUM, "src/qasm/elab.rs", "if self.whole || self.callouts { String::new() }", "if self.whole { String::new() }", ["--test", "review"], ["a_refusal_for_controls"], WASMQ),
    ("what is reported alone is a verdict", QUANTUM, "src/engine/job.rs", "    let c = check_text(text);\n    if alone(&c) {", "    let c = check_text(text);\n    if false && alone(&c) {", ["--features", "full", "--lib"], ["engine::job::tests::every_reason_reported_alone_is_a_verdict"]),
    ("a wasmQ job goes to no other verifier", QUANTUM, "src/engine/job.rs", "!super::wasmq::is_job(&job[\"result\"]) && (job", "(job", ["--test", "verify"], ["a_wasmq_job_naming_another_schema"], CLI),
    ("each line read on its own", CLI, "src/main.rs", "                    let c = job::check_bytes(line);", "                    let c = job::check_bytes(&bytes);", ["--test", "verify"], ["lines_read_each_line_on_its_own"]),
    ("a file of no line refused", CLI, "src/main.rs", "                if each.is_empty() {", "                if false {", ["--test", "verify"], ["lines_read_each_line_on_its_own"]),
    ("no envelope run for a job not judged", CLI, "src/main.rs", "                    if let Some(b) = &bundle\n                        && c.wasmq.is_some()", "                    if let Some(b) = &bundle", ["--test", "wasmq"], ["a_job_not_judged_is_reported_alone"]),
    ("a job not judged is said to be", CLI, "src/wasmq.rs", "    let Some(w) = c.wasmq.clone() else {", "    let Some(w) = c.wasmq.clone().or(Some(Value::Null)) else {", ["--test", "wasmq"], ["a_job_not_judged_is_reported_alone"]),
    # The one path every surface checks a job on (wai-quantum, engine::job).
    ("a bundle's contract compared as it is sealed", WASMQ, "src/run.rs", "canonical(&o.contract) == canonical(c_)", "o.contract == *c_", ["--test", "review"], ["a_contract_written_otherwise"]),
    ("a wasmQ job is held to its form on the one path", QUANTUM, "src/engine/job.rs", "    if super::wasmq::is_job(result) {", "    if false && super::wasmq::is_job(result) {", ["--features", "full", "--test", "wasmq_verdict"], ["every_reader", "a_wasmq_verdict"]),
    ("a broken form is a reason", QUANTUM, "src/engine/job.rs", "            c.push(WASMQ_FORM);", "", ["--features", "full", "--test", "wasmq_verdict"], ["every_reader"]),
    ("the command line's verdict for a wasmQ job is the one path's", CLI, "src/main.rs", "                    ok &= wasmq::report(&c, &v, bundle.as_deref());", "                    wasmq::report(&c, &v, bundle.as_deref());", ["--test", "wasmq"], ["json_and_lines_give", "a_job_is_read_from_its_text_as_every_job_is"]),
    ("a bundle that does not hold makes --json's verdict not valid", CLI, "src/wasmq.rs", "        said[\"valid\"] = json!(false);", "", ["--test", "wasmq"], ["json_and_lines_give"]),
]


def run(name, crate, path, old, new, args, tests, tested=None):
    p = os.path.join(crate, path)
    s = open(p).read()
    if s.count(old) != 1:
        return f"MISSING  {name}: the statement is in {p} {s.count(old)} times"
    # Where the tests are: a crate's own library tests, or the integration tests of wasmQ
    # or of the command line.
    # A crate's own tests run there too when the entry asks for its features.
    # An eighth member names it outright, for a file of one crate whose tests are another's.
    tested = tested or (crate if "--lib" in args or "--features" in args or crate == CLI else WASMQ)
    open(p, "w").write(s.replace(old, new))
    try:
        cmd = ["cargo", "test", "--release"] + args
        if tests:
            cmd += (["--"] if "--" not in args else []) + tests
        r = subprocess.run(cmd, cwd=tested, capture_output=True, text=True)
        out = r.stdout + r.stderr
        if "could not compile" in out:
            return f"SURVIVED {name}: it did not compile: " + next((l for l in out.splitlines() if l.startswith("error")), "")
        failed = [l.split()[1] for l in out.splitlines() if l.startswith("test ") and l.endswith("FAILED")]
        if failed:
            return f"KILLED   {name}: {', '.join(failed)}"
        if r.returncode != 0:
            return f"KILLED   {name}: the test process died ({r.returncode})"
        return f"SURVIVED {name}: every test passed"
    finally:
        open(p, "w").write(s)


def main():
    words = [a for a in sys.argv[1:] if a != "--check"]
    chosen = [m for m in M if not words or any(w in m[0] for w in words)]
    if "--check" in sys.argv:
        bad = [m[0] for m in chosen if open(os.path.join(m[1], m[2])).read().count(m[3]) != 1]
        print("\n".join(f"MISSING  {b}" for b in bad) or f"{len(chosen)} statements, each in the code once")
        sys.exit(1 if bad else 0)
    wrong = 0
    for m in chosen:
        line = run(*m)
        print(line, flush=True)
        wrong += not line.startswith("KILLED")
    print(f"{wrong} of {len(chosen)} not caught")
    sys.exit(1 if wrong else 0)


if __name__ == "__main__":
    main()