93 lines
3.5 KiB
Rust
93 lines
3.5 KiB
Rust
//! What an edit's evidence costs: every hand-written case of the edit eval
|
|
//! (`graph/edit/cases.json`) replayed through the edit loop with this machine's compiler, and
|
|
//! the cost of each bundle the loop gathered printed, part by part, with the spread over all.
|
|
//!
|
|
//! /usr/bin/time -l cargo run -q -p flowrs-core --example evidence_cost
|
|
//!
|
|
//! `time -l` gives the process tree's peak memory (the harness compiles and runs are children
|
|
//! of this one); each bundle states its own wall times and the budget it ran under. Run it on
|
|
//! the build host, never on the dev Mac: it compiles and runs a program per bundle.
|
|
use flowrs_core::graph::edit::{Turn, eval};
|
|
use flowrs_core::graph::verify::Native;
|
|
|
|
fn main() {
|
|
let reg = flowrs_core::graph::widget::builtin_registry();
|
|
let root = std::env::temp_dir().join(format!("flowrs_evidence_cost_{}", std::process::id()));
|
|
let tc = Native { root: root.clone() };
|
|
let mut totals: Vec<u64> = Vec::new();
|
|
let mut by_part: std::collections::BTreeMap<String, Vec<u64>> = Default::default();
|
|
for case in eval::cases().expect("the eval cases read") {
|
|
let Ok(mut s) = eval::session(&case, reg) else {
|
|
continue;
|
|
};
|
|
let mut i = 0;
|
|
while matches!(s.next(), Turn::Ask) {
|
|
let Some(r) = case.recorded.get(i) else {
|
|
break;
|
|
};
|
|
i += 1;
|
|
s.submit(&r.reply, &tc);
|
|
}
|
|
for (k, round) in s.rounds().iter().enumerate() {
|
|
let Some(b) = round
|
|
.verification
|
|
.as_ref()
|
|
.and_then(|v| v.bundle.as_deref())
|
|
else {
|
|
continue;
|
|
};
|
|
let (held, failed, skipped, not_run) = b.tally();
|
|
let parts: Vec<String> = b
|
|
.cost
|
|
.parts
|
|
.iter()
|
|
.map(|p| {
|
|
if let Some(ms) = p.wall_ms {
|
|
by_part.entry(p.part.clone()).or_default().push(ms);
|
|
}
|
|
format!(
|
|
"{} {}",
|
|
p.part,
|
|
p.wall_ms
|
|
.map_or_else(|| "-".into(), |ms| format!("{ms} ms"))
|
|
)
|
|
})
|
|
.collect();
|
|
if let Some(why) = b.failure() {
|
|
println!("{} round {}: FAILED: {why}", case.id, k + 1);
|
|
}
|
|
let total = b.cost.total_ms.unwrap_or_default();
|
|
totals.push(total);
|
|
println!(
|
|
"{} round {}: {total} ms | {} properties: {held} held, {failed} failed, {} not run | \
|
|
mutants {} of {} killed ({} sites) | {}",
|
|
case.id,
|
|
k + 1,
|
|
b.properties.len(),
|
|
skipped + not_run,
|
|
b.mutation.killed,
|
|
b.mutation.total,
|
|
b.mutation.sites,
|
|
parts.join(", ")
|
|
);
|
|
}
|
|
}
|
|
let spread = |v: &mut Vec<u64>| -> String {
|
|
if v.is_empty() {
|
|
return "none".into();
|
|
}
|
|
v.sort_unstable();
|
|
format!(
|
|
"n {}, median {} ms, p90 {} ms, max {} ms",
|
|
v.len(),
|
|
v[v.len() / 2],
|
|
v[(v.len() * 9 / 10).min(v.len() - 1)],
|
|
v[v.len() - 1]
|
|
)
|
|
};
|
|
println!("bundles: {}", spread(&mut totals));
|
|
for (part, mut v) in by_part {
|
|
println!(" {part}: {}", spread(&mut v));
|
|
}
|
|
let _ = std::fs::remove_dir_all(&root);
|
|
}
|