#!/usr/bin/env python3 """The checks of wasmQ, each removed in turn: the test that is of it must then fail. A test that still passes with its check removed cannot fail for that check, and says nothing of it. Each entry below names a check, where it is, how it is removed, and the tests that must then fail. Run from the repository's root, on a build host (it builds and tests a crate for each entry, so it takes a while): python3 wai/wai-wasmq/mutations.py every entry python3 wai/wai-wasmq/mutations.py WORD ... the entries whose name holds a WORD python3 wai/wai-wasmq/mutations.py --check only that each statement is in the code once Each file is restored after its entry, whatever happens. Exit 0 when every entry is caught: its tests ran and at least one failed, or the test process died. """ import os import subprocess import sys WASMQ = "wai/wai-wasmq" QUANTUM = "wai/wai-quantum" CLI = "wai/wai-quantum-cli" # (name, the crate of the file, the file in it, the statement, what it becomes, the cargo # test arguments, the tests, and optionally the crate whose tests they are). The tests are wasmQ's, but for a file of wai-quantum tested by # that crate's own tests (`--lib`, or a test of its features), and the command line's for a # file of the command line's. M = [ # The envelope (§2.3). ("the payload's magic", WASMQ, "src/envelope.rs", " if take(payload, &mut p, 4)? != MAGIC {", " if take(payload, &mut p, 4)? != MAGIC && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]), ("the contract's canonical form", WASMQ, "src/envelope.rs", " if canonical(&contract) != text {", " if canonical(&contract) != text && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]), ("the frames end with the payload", WASMQ, "src/envelope.rs", " if p != payload.len() {", " if p != payload.len() && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]), ("the contract's members", WASMQ, "src/envelope.rs", " if canonical(&self::contract(&bundle, &method)) != text {", " if canonical(&self::contract(&bundle, &method)) != text && false {", ["--test", "vectors"], ["each_refusal_is_refused"]), ("the manifest is the contract's", WASMQ, "src/envelope.rs", " if manifest_bytes != self::manifest(&bundle, h.reading.program.n_qubits)?.as_slice() {", " if manifest_bytes != self::manifest(&bundle, h.reading.program.n_qubits)?.as_slice() && false {", ["--test", "adversarial"], ["each_rule_of_the_envelope"]), ("a frame's digest", WASMQ, "src/envelope.rs", " if got != want {", " if got != want && false {", ["--test", "adversarial"], ["a_changed_module_or_program", "a_prior_whose_hash"]), ("an input a double holds", WASMQ, "src/envelope.rs", "v.unsigned_abs() > MAX_EXACT) {", "v.unsigned_abs() > MAX_EXACT && false) {", ["--test", "review"], ["an_input_a_double_cannot_hold"]), ("one callout for each extern", WASMQ, "src/envelope.rs", " if b.callouts.len() != p.externs.len() || b.callouts.iter().zip(&p.externs).any(|(c, d)| c.name != d.name) {", " if false {", ["--test", "review"], ["a_contract_names_one_callout"]), ("the memory a host reaches", WASMQ, "src/envelope.rs", " if f.memory.is_none() || !f.memory_exported {", " if f.memory.is_none() {", ["--test", "review"], ["a_bundle_whose_memory_the_host_cannot_reach"]), ("a module with priors exports cabi_realloc", WASMQ, "src/envelope.rs", ' reachable(f, &m.name, "the host passes a prior", true)?;', ' reachable(f, &m.name, "the host passes a prior", false)?;', ["--test", "review"], ["a_module_with_priors_exports"]), ("the Clifford gates", WASMQ, "src/envelope.rs", " if let Some(op) = p.ops.iter().find(", " if let Some(op) = p.ops.iter().take(0).find(", ["--test", "review"], ["a_program_outside_the_clifford_gates"]), # What a run asks of its host (§4.7, §4.8). ("an envelope's bytes", WASMQ, "src/envelope.rs", " if bytes.len() > MAX_ENVELOPE_BYTES {", " if bytes.len() > MAX_ENVELOPE_BYTES && false {", ["--test", "sweep"], ["an_envelope_past"]), ("an envelope's frames", WASMQ, "src/envelope.rs", " if frames >= MAX_ENVELOPE_BYTES as u64 {", " if frames >= MAX_ENVELOPE_BYTES as u64 && false {", ["--test", "sweep"], ["an_envelope_past"]), ("a program's text", WASMQ, "src/envelope.rs", " if program.len() > MAX_PROGRAM_BYTES {", " if program.len() > MAX_PROGRAM_BYTES && false {", ["--test", "reading"], ["a_program_past_the_bounds"]), ("the code among a bundle's modules", WASMQ, "src/envelope.rs", " if code > MAX_CODE_BYTES {", " if code > MAX_CODE_BYTES && false {", ["--test", "sweep"], ["the_code_among_a_bundle"]), ("the memory among a bundle's modules", WASMQ, "src/envelope.rs", " if pages > profile::MAX_MEMORY_PAGES {", " if pages > profile::MAX_MEMORY_PAGES && false {", ["--test", "sweep"], ["the_modules_of_a_bundle"]), ("a memory's maximum, not its start", WASMQ, "src/envelope.rs", ".map(|(_, maximum)| maximum).sum();", ".map(|(initial, _)| initial.min(1)).sum();", ["--test", "sweep"], ["the_modules_of_a_bundle"]), ("a run's results", WASMQ, "src/envelope.rs", " if bits > MAX_RESULT_BITS {", " if bits > MAX_RESULT_BITS && false {", ["--test", "sweep"], ["a_bundle_whose_result"]), ("a run's results, at the limit", WASMQ, "src/envelope.rs", " if bits > MAX_RESULT_BITS {", " if bits >= MAX_RESULT_BITS {", ["--test", "sweep"], ["a_bundle_whose_result"]), ("a run's work", WASMQ, "src/envelope.rs", " if work > MAX_HOST_WORK {", " if work > MAX_HOST_WORK && false {", ["--test", "sweep"], ["a_run_no_host", "the_work_limit"]), ("a run's work, at the limit", WASMQ, "src/envelope.rs", " if work > MAX_HOST_WORK {", " if work >= MAX_HOST_WORK {", ["--test", "sweep"], ["the_work_limit", "a_run_s_fuel_is_bounded"]), ("a gate's work", WASMQ, "src/envelope.rs", " Step::Instr(Instr::Gate(_)) => r,", " Step::Instr(Instr::Gate(_)) => 1,", ["--test", "sweep"], ["the_work_a_run_asks"]), ("a measurement's work", WASMQ, "src/envelope.rs", " Step::Instr(Instr::Measure { .. } | Instr::Reset(_)) => MEASURE_WORK * n * r,", " Step::Instr(Instr::Measure { .. } | Instr::Reset(_)) => n * r,", ["--test", "sweep"], ["the_work_a_run_asks", "a_run_no_host"]), ("a tableau's work", WASMQ, "src/envelope.rs", " let mut shot = 2 * n * r;", " let mut shot = 0;", ["--test", "sweep"], ["the_work_a_run_asks"]), ("the calls of a callout", WASMQ, "src/envelope.rs", " CALL_WORK * (1 + lists + u128::from(post && crate::abi::uses_memory(decl)))", " CALL_WORK", ["--test", "sweep"], ["a_callout_s_work"]), ("the bits of a callout", WASMQ, "src/envelope.rs", " + BIT_WORK * (input.len() + output.len()) as u128 + 1", " + 1", ["--test", "sweep"], ["a_callout_s_work", "the_work_a_run_asks"]), ("the calls that make an instance", WASMQ, "src/envelope.rs", " let calls = 1 + 2 * m.priors.len() + ", " let calls = 1 + m.priors.len() + ", ["--test", "sweep"], ["a_callout_s_work", "the_work_a_run_asks"]), ("an instance's memory", WASMQ, "src/envelope.rs", ".map_or(0, |(initial, _)| u128::from(initial)) * 8192;", ".map_or(0, |(initial, _)| u128::from(initial));", ["--test", "sweep"], ["the_work_a_run_asks", "memory_a_module_grows"]), ("one instance for a run of `job`", WASMQ, "src/envelope.rs", ' let instances = if b.run.state == "shot" { b.run.shots } else { 1 };', " let instances = b.run.shots;", ["--test", "sweep"], ["the_work_a_run_asks", "a_run_no_host"]), ("a module's code in the work", WASMQ, "src/envelope.rs", " code += CODE_WORK * m.bytes.len() as u128;", "", ["--test", "sweep"], ["the_work_a_run_asks"]), ("the fuel in the work", WASMQ, "src/envelope.rs", " + code + FUEL_WORK * u128::from(b.run.per_job)", " + code", ["--test", "sweep"], ["a_run_s_fuel_is_bounded"]), # The reader's bounds (§4.8), and its nesting. ("the reader's work", QUANTUM, "src/qasm/elab.rs", " if self.reading > self.bounds.reading {\n return self.fail(", " if self.reading > self.bounds.reading && false {\n return self.fail(", ["--test", "reading"], ["a_program_past_the_bounds"]), ("the steps of a statement", QUANTUM, "src/qasm/elab.rs", " if self.steps.len().saturating_add(more) > self.bounds.steps {", " if self.steps.len().saturating_add(more) > self.bounds.steps && false {", ["--test", "reading"], ["a_program_past_the_bounds"]), ("the steps of a reading", QUANTUM, "src/qasm/elab.rs", " if steps > bounds.steps {", " if steps > bounds.steps && false {", ["--test", "sweep"], ["the_steps_of_a_program"]), ("the bits of a program", QUANTUM, "src/qasm/elab.rs", " if held > u64::from(self.bounds.bits) {", " if held > u64::from(self.bounds.bits) && false {", ["--test", "sweep"], ["a_bundle_whose_result"]), ("the bits of the calls", QUANTUM, "src/qasm/elab.rs", " if self.callout_bits > self.bounds.callout_bits {", " if self.callout_bits > self.bounds.callout_bits && false {", ["--test", "reading"], ["a_program_past_the_bounds"]), ("an array element's bits", QUANTUM, "src/qasm/elab.rs", " Ty::Bit(n) => u64::from(n.unwrap_or(1)),", " Ty::Bit(_) => 1u64,", ["--test", "reading"], ["a_program_past_the_bounds"]), ("an array's elements", QUANTUM, "src/qasm/elab.rs", " if elements > u64::from(self.bounds.bits) {", " if elements > u64::from(self.bounds.bits) && false {", ["--test", "reading"], ["a_program_past_the_bounds"]), ("the qubits a program declares", QUANTUM, "src/qasm/elab.rs", " if total > u64::from(self.bounds.qubits) {", " if total > u64::from(self.bounds.qubits) && false {", ["--test", "reading"], ["a_program_past_the_bounds"]), ("the qubits a program names", QUANTUM, "src/qasm/elab.rs", " if m >= MAX_QUBITS || m >= self.bounds.qubits {", " if m >= MAX_QUBITS {", ["--test", "reading"], ["a_program_past_the_bounds"]), ("the nesting", QUANTUM, "src/qasm/parse.rs", " if self.depth > super::MAX_NESTING {", " if self.depth > super::MAX_NESTING && false {", ["--test", "reading"], ["a_program_past_the_bounds"]), ("no measurement added to a hybrid program", QUANTUM, "src/qasm/elab.rs", " if measured.is_empty() && !self.callouts {", " if measured.is_empty() {", ["--test", "vectors"], ["the_vectors_are"]), # Lowering and the lifecycle (§5). ("a callout's width", WASMQ, "src/abi.rs", " if n < 64 && v >> n != 0 {", " if n < 64 && v >> n != 0 && false {", ["--test", "adversarial"], ["a_trap_and_a_result_past_its_width"]), ("a signed argument, sign-extended", WASMQ, "src/abi.rs", " && v >> (w - 1) & 1 == 1", " && false", ["--test", "vectors"], ["the_vectors_are"]), ("a list result is 4-aligned", WASMQ, "src/abi.rs", " if !ret.is_multiple_of(4) {", " if false {", ["--test", "vectors"], ["the_vectors_are"]), ("cabi_post after a list result", WASMQ, "src/abi.rs", " if has_post {", " if false {", ["--test", "vectors"], ["the_vectors_are"]), # The engines (§4). ("a body ends", WASMQ, "src/profile.rs", " ops.finish().map_err(malformed)?;", "", ["--test", "review"], ["a_body_that_does_not_end"]), ("a frame's operands", WASMQ, "src/profile.rs", " operands = operands.max(fv.operand_stack_height());", "", ["--test", "engines"], ["a_native_frame"]), ("a frame's operands, in the interpreter", WASMQ, "src/interp.rs", " operands = operands.max(fv.operand_stack_height());", "", ["--test", "engines"], ["a_native_frame"]), ("the stack before an indirect call", WASMQ, "src/interp.rs", " self.suspend(f.weight)?;\n let i = self.pop() as u32 as usize;", " let i = self.pop() as u32 as usize;", ["--test", "review"], ["an_indirect_call_where_the_stack_ends"]), ("the engine reaches only an exported memory", WASMQ, "src/interp.rs", " self.m.memory_exported.then_some(&mut self.mem[..])", " Some(&mut self.mem[..])", ["--test", "review"], ["every_engine_reaches_a_module_s_memory"]), ("an argument is of its parameter's type", WASMQ, "src/interp.rs", "if matches!(t, ValType::I32 | ValType::F32) { a as u32 as u64 } else { a }", "if matches!(t, ValType::I32 | ValType::F32) { a } else { a }", ["--test", "review"], ["an_argument_is_read_as_a_value"]), ("a NaN is made canonical by the interpreter", WASMQ, "src/interp.rs", "fn c64(x: f64) -> u64 {\n if x.is_nan() { CANON64 } else { x.to_bits() }", "fn c64(x: f64) -> u64 {\n if false { CANON64 } else { x.to_bits() }", ["--test", "fuzz"], ["a_changed_module_is_refused"]), ("a page grown, in the interpreter's fuel", WASMQ, "src/interp.rs", " self.charge(delta * meter::PAGE_FUEL)?;", " self.charge(delta)?;", ["--test", "sweep"], ["memory_a_module_grows"]), ("a page grown, in the prepared module's fuel", WASMQ, "src/meter.rs", "if matches!(op, Operator::MemoryGrow { .. }) { PAGE_FUEL } else { 1 }", "1", ["--test", "engines"], ["the_fuel_is_the_same"]), ("a run's fuel is charged at its start", WASMQ, "src/meter.rs", " charge_const(&mut f, &g, cost);", "", ["--test", "fuzz"], ["a_changed_module_is_refused"]), ("the frame weight in the prepared module", WASMQ, "src/meter.rs", " stack_in(&mut f, &g, weight_here);", " stack_in(&mut f, &g, 1);", ["--test", "engines"], ["the_stack_ends_at_the_same_call"]), ("the function limit", WASMQ, "src/profile.rs", ' Payload::FunctionSection(r) => count(r.count(), "functions", MAX_ITEMS)?,', " Payload::FunctionSection(_) => {}", ["--test", "review"], ["the_function_limit"]), # The tableau: a determined outcome, word by word. ("the sign of a determined outcome", QUANTUM, "src/quantum_tableau.rs", " let plus = (px & pz & z & !x) | (px & !pz & z & x) | (!px & pz & x & !z);", " let plus = (px & pz & z & !x) | (!px & pz & x & !z);", ["--features", "full", "--lib", "--", "quantum_tableau::tests::a_determined_measurement"], []), ("a determined outcome's carry", QUANTUM, "src/quantum_tableau.rs", " if (x.count_ones() & 1) == 1 {\n cx = !cx;\n }", "", ["--features", "full", "--lib", "--", "quantum_tableau::tests::a_determined_measurement"], []), # A job (§6.5). ("the host's unsealed status, noted", QUANTUM, "src/engine/wasmq.rs", ' "status" if value == body_status(r) => {}', ' "status" => {}', ["--test", "review"], ["a_job_whose_status_or_method"]), ("verify's read, bounded by size", CLI, "src/main.rs", " if f.metadata().map_err(|e| format!(\"{}: {e}\", p.display()))?.len() > MAX_JOB_BYTES {", " if false {", ["--test", "wasmq"], ["verify_refuses_a_file_past"]), ("the signer is the receipt's key", WASMQ, "src/run.rs", 'signer: seal["key"].as_str().map(str::to_string)', "signer: None", ["--test", "review"], ["a_job_names_its_signer"]), ("the signature is the engine's check of the seal", WASMQ, "src/run.rs", " signature: !c.reasons.iter().any(|x| sealing.contains(x)),", " signature: true,", ["--test", "adversarial"], ["a_receipt_changed_after_sealing"]), ("a wasmQ job sealed in the canonical form", QUANTUM, "src/engine/wasmq.rs", ' if r["receipt"]["canon"] != "rfc8785" {', " if false {", ["--features", "full", "--test", "wasmq_verdict"], ["every_reader"]), ("the result's members", QUANTUM, "src/engine/wasmq.rs", " members(r, \"the result\", &RESULT, &[\"fault\"], &mut out);", "", ["--test", "sweep"], ["a_job_whose_result"]), ("the contract's form", QUANTUM, "src/engine/wasmq.rs", " contract_form(c, &mut out);", "", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]), ("the contract's method", QUANTUM, "src/engine/wasmq.rs", " if run[\"method\"] != METHOD {", " if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]), ("the contract's shots", QUANTUM, "src/engine/wasmq.rs", " if !whole(&run[\"shots\"], 1, MAX_SHOTS) {", " if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]), ("what a result repeats from its contract", QUANTUM, "src/engine/wasmq.rs", " if !same(&r[name], &want) {", " if !same(&r[name], &want) && false {", ["--test", "sweep"], ["a_job_whose_result"]), ("fuel used within the run's budget", QUANTUM, "src/engine/wasmq.rs", " if count(&fuel[\"used\"]).is_none_or(|u| u > per_job) {", " if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]), ("a call within the call's budget", QUANTUM, "src/engine/wasmq.rs", " if max.is_none_or(|m| m > per_call) {", " if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]), ("a callout's fuel within its calls", QUANTUM, "src/engine/wasmq.rs", " if all.is_none_or(|a| n.zip(max).is_none_or(", " if false && all.is_none_or(|a| n.zip(max).is_none_or(", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]), ("the instances' fuel", QUANTUM, "src/engine/wasmq.rs", " if count(&fuel[\"instances\"]).is_none_or(|n| u128::from(n) > made.saturating_mul(stretches).saturating_mul(u128::from(per_instance))) {", " if false {", ["--test", "sweep"], ["a_job_whose_contract_or_receipt"]), ("a fault's fuel within its phase's budget", QUANTUM, "src/engine/wasmq.rs", " if used.is_none_or(|n| n > budget) {", " if false {", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_fault_outside_a_call_is_within_the_instance_s_budget"]), ("the fuel adds up", QUANTUM, "src/engine/wasmq.rs", " if spent.is_none() || spent != count(&fuel[\"used\"]) {", " if (spent.is_none() || spent != count(&fuel[\"used\"])) && false {", ["--test", "sweep"], ["a_job_whose_result"]), ("a fault after every shot", QUANTUM, "src/engine/wasmq.rs", " if shots_run.zip(shots).is_none_or(|(ran, all)| ran >= all) {", " if false {", ["--test", "sweep"], ["a_job_whose_result"]), ("the counts' sum", QUANTUM, "src/engine/wasmq.rs", " if total.is_none() || total != shots_run {", " if (total.is_none() || total != shots_run) && false {", ["--test", "sweep"], ["a_job_whose_result"]), ("a record's shots are the run's", WASMQ, "src/run.rs", " if call.shot >= b.run.shots {", " if call.shot >= b.run.shots && false {", ["--test", "sweep"], ["a_record_of_a_shot"]), ("a record's shots are in order", WASMQ, "src/run.rs", " if call.shot < last {", " if call.shot < last && false {", ["--test", "sweep"], ["a_record_of_a_shot"]), # The command line. ("a job read from its text", CLI, "src/main.rs", " } else if as_json || (v.get(\"request\").is_some() && v.get(\"result\").is_some()) {\n let c = job::check_text(&src);", " } else if as_json || (v.get(\"request\").is_some() && v.get(\"result\").is_some()) {\n let c = job::check(&v);", ["--test", "wasmq"], ["a_job_is_read_from_its_text_as_every_job_is"]), ("no history for a wasmQ job", CLI, "src/main.rs", " if wasmq && hist.is_some() {", " if false {", ["--test", "wasmq"], ["json_and_lines_give"]), # Round three: a number past 2^53 - 1 is no count; a stopped call is one of the calls # made; a budget is at most 2^39 when packed; wasmQ's reading offers no method it cannot # use; what is reported alone is a verdict; routing, lines and the not-judged report. ("a whole number past 2^53 - 1 is no count", QUANTUM, "src/engine/wasmq.rs", "v.as_u64().filter(|&n| n <= MAX_EXACT).or_else(", "v.as_u64().or_else(", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_number_is_read_by_its_value", "engine::wasmq::tests::a_budget_past_two_to_the_53"]), ("a stopped call the fuel counts", QUANTUM, "src/engine/wasmq.rs", "None => out.push(format!(\"a call of {name:?} that stopped, where the fuel counts no call of it\")),", "None => {}", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_call_s_fault_is_one_of_the_calls_made"]), ("a stopped call within its callout's largest", QUANTUM, "src/engine/wasmq.rs", "Some(m) if count(&m[\"max\"]).zip(used).is_none_or(|(max, u)| u > max) =>", "Some(m) if false =>", ["--features", "full", "--lib"], ["engine::wasmq::tests::a_call_s_fault_is_one_of_the_calls_made"]), ("a budget at most 2^39 when packed", WASMQ, "src/envelope.rs", " if v > MAX_BUDGET {", " if v > MAX_EXACT {", ["--test", "review"], ["a_budget_past_what_a_run_may_be_given"]), ("a hybrid reading offers no other method", QUANTUM, "src/qasm/elab.rs", "if self.whole || self.callouts { String::new() }", "if self.whole { String::new() }", ["--test", "review"], ["a_refusal_for_controls"], WASMQ), ("what is reported alone is a verdict", QUANTUM, "src/engine/job.rs", " let c = check_text(text);\n if alone(&c) {", " let c = check_text(text);\n if false && alone(&c) {", ["--features", "full", "--lib"], ["engine::job::tests::every_reason_reported_alone_is_a_verdict"]), ("a wasmQ job goes to no other verifier", QUANTUM, "src/engine/job.rs", "!super::wasmq::is_job(&job[\"result\"]) && (job", "(job", ["--test", "verify"], ["a_wasmq_job_naming_another_schema"], CLI), ("each line read on its own", CLI, "src/main.rs", " let c = job::check_bytes(line);", " let c = job::check_bytes(&bytes);", ["--test", "verify"], ["lines_read_each_line_on_its_own"]), ("a file of no line refused", CLI, "src/main.rs", " if each.is_empty() {", " if false {", ["--test", "verify"], ["lines_read_each_line_on_its_own"]), ("no envelope run for a job not judged", CLI, "src/main.rs", " if let Some(b) = &bundle\n && c.wasmq.is_some()", " if let Some(b) = &bundle", ["--test", "wasmq"], ["a_job_not_judged_is_reported_alone"]), ("a job not judged is said to be", CLI, "src/wasmq.rs", " let Some(w) = c.wasmq.clone() else {", " let Some(w) = c.wasmq.clone().or(Some(Value::Null)) else {", ["--test", "wasmq"], ["a_job_not_judged_is_reported_alone"]), # The one path every surface checks a job on (wai-quantum, engine::job). ("a bundle's contract compared as it is sealed", WASMQ, "src/run.rs", "canonical(&o.contract) == canonical(c_)", "o.contract == *c_", ["--test", "review"], ["a_contract_written_otherwise"]), ("a wasmQ job is held to its form on the one path", QUANTUM, "src/engine/job.rs", " if super::wasmq::is_job(result) {", " if false && super::wasmq::is_job(result) {", ["--features", "full", "--test", "wasmq_verdict"], ["every_reader", "a_wasmq_verdict"]), ("a broken form is a reason", QUANTUM, "src/engine/job.rs", " c.push(WASMQ_FORM);", "", ["--features", "full", "--test", "wasmq_verdict"], ["every_reader"]), ("the command line's verdict for a wasmQ job is the one path's", CLI, "src/main.rs", " ok &= wasmq::report(&c, &v, bundle.as_deref());", " wasmq::report(&c, &v, bundle.as_deref());", ["--test", "wasmq"], ["json_and_lines_give", "a_job_is_read_from_its_text_as_every_job_is"]), ("a bundle that does not hold makes --json's verdict not valid", CLI, "src/wasmq.rs", " said[\"valid\"] = json!(false);", "", ["--test", "wasmq"], ["json_and_lines_give"]), ] def run(name, crate, path, old, new, args, tests, tested=None): p = os.path.join(crate, path) s = open(p).read() if s.count(old) != 1: return f"MISSING {name}: the statement is in {p} {s.count(old)} times" # Where the tests are: a crate's own library tests, or the integration tests of wasmQ # or of the command line. # A crate's own tests run there too when the entry asks for its features. # An eighth member names it outright, for a file of one crate whose tests are another's. tested = tested or (crate if "--lib" in args or "--features" in args or crate == CLI else WASMQ) open(p, "w").write(s.replace(old, new)) try: cmd = ["cargo", "test", "--release"] + args if tests: cmd += (["--"] if "--" not in args else []) + tests r = subprocess.run(cmd, cwd=tested, capture_output=True, text=True) out = r.stdout + r.stderr if "could not compile" in out: return f"SURVIVED {name}: it did not compile: " + next((l for l in out.splitlines() if l.startswith("error")), "") failed = [l.split()[1] for l in out.splitlines() if l.startswith("test ") and l.endswith("FAILED")] if failed: return f"KILLED {name}: {', '.join(failed)}" if r.returncode != 0: return f"KILLED {name}: the test process died ({r.returncode})" return f"SURVIVED {name}: every test passed" finally: open(p, "w").write(s) def main(): words = [a for a in sys.argv[1:] if a != "--check"] chosen = [m for m in M if not words or any(w in m[0] for w in words)] if "--check" in sys.argv: bad = [m[0] for m in chosen if open(os.path.join(m[1], m[2])).read().count(m[3]) != 1] print("\n".join(f"MISSING {b}" for b in bad) or f"{len(chosen)} statements, each in the code once") sys.exit(1 if bad else 0) wrong = 0 for m in chosen: line = run(*m) print(line, flush=True) wrong += not line.startswith("KILLED") print(f"{wrong} of {len(chosen)} not caught") sys.exit(1 if wrong else 0) if __name__ == "__main__": main()