2b6f30d348
Built alone from an empty CARGO_HOME, its tests passed, and flowrsc checked, ran and exported a document from the conformance corpus; the export built alone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
539 lines
21 KiB
Rust
539 lines
21 KiB
Rust
//! Branch row 2's acceptance (work/ledger), offline, on the eval set's scripted replies.
|
|
//!
|
|
//! - **Quality.** The `Ledger` stage reaches 100% coverage on the eval set: every value or
|
|
//! widget a patch chose that its request did not name is listed, derived here again from
|
|
//! each diff. Every rationale sentence names only changed handles. A ledger that drops an
|
|
//! entry or cites an unchanged handle is refused. Explanations anchor to the right Rust
|
|
//! lines for all 19 corpus graphs, checked against where the compiled program binds each
|
|
//! node's outputs (the probes the tier check reads).
|
|
//! - **Ownership.** Any ledger entry reopens as an alternative set: the same edit with each
|
|
//! alternative written in, every one verified. Six deliberately underspecified requests
|
|
//! are answered with a clarifying question, not a patch, and no model is asked.
|
|
//!
|
|
//! The browser's checks (no compiler) keep this fast: the ledger does not depend on the
|
|
//! compile stages, and `tests/agent_edit_eval.rs` runs the same eval, guard included, with
|
|
//! cargo.
|
|
|
|
use flowrs_core::codegen::rust;
|
|
use flowrs_core::graph::corpus;
|
|
use flowrs_core::graph::dsl;
|
|
use flowrs_core::graph::edit::ledger::{self, Said};
|
|
use flowrs_core::graph::edit::{self, Options, Outcome, Proposal, Session, Turn, eval, variants};
|
|
use flowrs_core::graph::ir::{Graph, NodeId};
|
|
use flowrs_core::graph::verify::{self, InBrowser, Stage, Status};
|
|
use flowrs_core::graph::widget::builtin_registry;
|
|
|
|
/// A case replayed with the browser's checks: its outcome and the session that ran it.
|
|
fn replayed(c: &eval::Case, opts: Option<Options>) -> (Outcome, usize) {
|
|
let mut s = match opts {
|
|
Some(o) => Session::new(
|
|
eval::base_graph(c).unwrap(),
|
|
builtin_registry(),
|
|
&c.request,
|
|
eval::expect_of(c).unwrap(),
|
|
o,
|
|
),
|
|
None => eval::session(c, builtin_registry()).unwrap(),
|
|
};
|
|
let mut i = 0usize;
|
|
while let Turn::Ask = s.next() {
|
|
let Some(r) = c.recorded.get(i) else {
|
|
panic!("{}: the loop wanted reply {}", c.id, i + 1)
|
|
};
|
|
s.submit(&r.reply, &InBrowser);
|
|
i += 1;
|
|
}
|
|
let Turn::Done(o) = s.next() else {
|
|
unreachable!()
|
|
};
|
|
(*o, i)
|
|
}
|
|
|
|
fn is_ask(c: &eval::Case) -> bool {
|
|
c.properties
|
|
.iter()
|
|
.any(|p| matches!(p, eval::Property::Asks(_)))
|
|
}
|
|
|
|
#[test]
|
|
fn the_ledger_covers_every_unasked_choice_on_the_eval_set() {
|
|
let cases = eval::cases().unwrap();
|
|
let mut tally = eval::LedgerTally::default();
|
|
let (mut proposals, mut entries, mut consequential, mut defaults) = (0, 0, 0, 0);
|
|
let mut lines = Vec::new();
|
|
for c in &cases {
|
|
let (o, _) = replayed(c, None);
|
|
for p in o.proposals() {
|
|
proposals += 1;
|
|
let (why, t) = eval::ledger_guard(p, &c.request);
|
|
assert!(why.is_empty(), "{}: {why:?}", c.id);
|
|
assert!(
|
|
matches!(
|
|
p.verification.status(Stage::Ledger),
|
|
Some(Status::Passed(_))
|
|
),
|
|
"{}",
|
|
c.id
|
|
);
|
|
entries += p.ledger.len();
|
|
consequential += p.ledger.iter().filter(|e| e.consequential).count();
|
|
defaults += p.ledger.iter().filter(|e| e.default).count();
|
|
tally.add(&t);
|
|
lines.push(format!(
|
|
" {:<34} {:>2} listed of {:>2} · {} stated · {} sentence(s)",
|
|
c.id, t.listed, t.required, t.stated, t.sentences
|
|
));
|
|
}
|
|
}
|
|
println!(
|
|
"ledger over the eval set: {} proposals; {} of {} unasked choices listed ({:.1}%), \
|
|
{} with the agent's reasons; {} consequential, {} left at a default; {} rationale \
|
|
sentences, {} naming an unchanged handle\n{}",
|
|
proposals,
|
|
tally.listed,
|
|
tally.required,
|
|
100.0 * tally.listed as f64 / tally.required.max(1) as f64,
|
|
tally.stated,
|
|
consequential,
|
|
defaults,
|
|
tally.sentences,
|
|
tally.uncited,
|
|
lines.join("\n")
|
|
);
|
|
assert_eq!(tally.listed, tally.required, "coverage is not 100%");
|
|
assert_eq!(entries, tally.required, "an entry for something not chosen");
|
|
assert_eq!(tally.uncited, 0);
|
|
assert!(
|
|
tally.required >= 40,
|
|
"a set that exercises the ledger: {}",
|
|
tally.required
|
|
);
|
|
assert!(tally.stated >= 3, "the agent's own reasons are kept");
|
|
assert!(tally.sentences >= 60);
|
|
}
|
|
|
|
/// The ledger the agent wrote is not taken on trust: a claim the diff does not bear out is
|
|
/// said as a warning and left out, and a reason that names an unchanged node is replaced.
|
|
#[test]
|
|
fn what_the_agent_says_is_checked_against_the_diff() {
|
|
let c = eval::cases()
|
|
.unwrap()
|
|
.into_iter()
|
|
.find(|c| c.id == "open-filter-less-ringing")
|
|
.unwrap();
|
|
let (o, _) = replayed(&c, None);
|
|
let Outcome::Alternatives(ps) = o else {
|
|
panic!()
|
|
};
|
|
let fam = &ps[0];
|
|
let e = fam
|
|
.ledger
|
|
.iter()
|
|
.find(|e| e.key.as_deref() == Some("family"))
|
|
.unwrap();
|
|
assert!(e.stated && e.reason.contains("group delay"), "{e:?}");
|
|
assert_eq!(
|
|
e.alternatives[..2],
|
|
["butterworth".to_string(), "chebyshev1".to_string()]
|
|
);
|
|
// Its `why` named n2 (the FFT), which the edit does not change.
|
|
assert!(fam.rationale.iter().all(|r| !r.stated));
|
|
assert!(
|
|
fam.verification
|
|
.findings
|
|
.iter()
|
|
.any(|f| f.stage == Stage::Ledger && !f.error && f.message.contains("names n2")),
|
|
"{:?}",
|
|
fam.verification.findings
|
|
);
|
|
let seven = eval::cases()
|
|
.unwrap()
|
|
.into_iter()
|
|
.find(|c| c.id == "sum-prints-seven")
|
|
.unwrap();
|
|
let (o, _) = replayed(&seven, None);
|
|
let Outcome::Verified(p) = o else { panic!() };
|
|
assert_eq!(p.ledger.len(), 1);
|
|
assert_eq!(p.ledger[0].reason, "2 + 5 is 7");
|
|
assert!(p.rationale[0].stated);
|
|
assert!(p.rationale[0].text.contains("sum the print shows is 7"));
|
|
}
|
|
|
|
/// Mutations of a passing ledger: each one the stage, and the eval's guard, refuse.
|
|
#[test]
|
|
fn a_ledger_that_drops_an_entry_or_cites_an_unchanged_handle_is_refused() {
|
|
let c = eval::cases()
|
|
.unwrap()
|
|
.into_iter()
|
|
.find(|c| c.id == "sum-prints-seven")
|
|
.unwrap();
|
|
let (o, _) = replayed(&c, None);
|
|
let Outcome::Verified(p) = o else { panic!() };
|
|
assert!(!p.ledger.is_empty());
|
|
let said = Said::new(&p.asked);
|
|
let stage = |p: &Proposal| {
|
|
let mut v = p.verification.clone();
|
|
verify::check_ledger(
|
|
&mut v,
|
|
&said,
|
|
&p.graph,
|
|
&p.changes,
|
|
&verify::LedgerClaim {
|
|
ledger: &p.ledger,
|
|
rationale: &p.rationale,
|
|
notes: &[],
|
|
},
|
|
builtin_registry(),
|
|
);
|
|
(
|
|
matches!(v.status(Stage::Ledger), Some(Status::Passed(_))),
|
|
v.findings
|
|
.iter()
|
|
.filter(|f| f.stage == Stage::Ledger && f.error)
|
|
.map(|f| f.message.clone())
|
|
.collect::<Vec<_>>(),
|
|
)
|
|
};
|
|
assert!(stage(&p).0, "{:?}", stage(&p).1);
|
|
|
|
let mut dropped = p.clone();
|
|
dropped.ledger.remove(0);
|
|
let (ok, why) = stage(&dropped);
|
|
assert!(
|
|
!ok && why.iter().any(|m| m.contains("does not list")),
|
|
"{why:?}"
|
|
);
|
|
assert!(!eval::ledger_guard(&dropped, &c.request).0.is_empty());
|
|
// The eval grades a case down for it, whatever else the case asks.
|
|
let base = eval::base_graph(&c).unwrap();
|
|
assert!(eval::grade(&c, &Outcome::Verified(p.clone()), 1, &base).passed);
|
|
let graded = eval::grade(&c, &Outcome::Verified(dropped.clone()), 1, &base);
|
|
assert!(!graded.passed, "{graded:?}");
|
|
|
|
let mut cites = p.clone();
|
|
cites.rationale[0].text = "It reads n0's signal.".into();
|
|
let (ok, why) = stage(&cites);
|
|
assert!(!ok && why.iter().any(|m| m.contains("names n0")), "{why:?}");
|
|
assert!(!eval::ledger_guard(&cites, &c.request).0.is_empty());
|
|
|
|
let mut wrong = p.clone();
|
|
wrong.ledger[0].value = "elsewhere".into();
|
|
assert!(!stage(&wrong).0);
|
|
|
|
let mut extra = p.clone();
|
|
let mut e = extra.ledger[0].clone();
|
|
e.handle = "n0".into();
|
|
extra.ledger.push(e);
|
|
assert!(!stage(&extra).0);
|
|
|
|
let mut short = p.clone();
|
|
short.rationale.pop();
|
|
let (ok, why) = stage(&short);
|
|
assert!(
|
|
!ok && why.iter().any(|m| m.contains("rationales")),
|
|
"{why:?}"
|
|
);
|
|
}
|
|
|
|
/// Six requests that leave three or more consequential choices open are answered with a
|
|
/// question, before any model is asked; with asking turned off, the scripted patch each
|
|
/// would otherwise have shown verifies and makes at least that many choices nobody asked
|
|
/// for — the question was warranted.
|
|
#[test]
|
|
fn six_underspecified_requests_get_a_question_not_a_patch() {
|
|
let cases: Vec<eval::Case> = eval::cases().unwrap().into_iter().filter(is_ask).collect();
|
|
assert_eq!(cases.len(), 6);
|
|
let mut lines = Vec::new();
|
|
for c in &cases {
|
|
let (o, calls) = replayed(c, None);
|
|
let Outcome::Answered {
|
|
text,
|
|
question: Some(q),
|
|
} = &o
|
|
else {
|
|
panic!("{}: {o:?}", c.id)
|
|
};
|
|
assert_eq!(calls, 0, "{}: a model was asked", c.id);
|
|
assert!(q.open.len() >= ledger::ASK_AT, "{}: {q:?}", c.id);
|
|
assert_eq!(*text, q.text);
|
|
let r = eval::grade(c, &o, 0, &eval::base_graph(c).unwrap());
|
|
assert!(r.passed, "{}: {:?}", c.id, r.failures);
|
|
|
|
let (silent, _) = replayed(
|
|
c,
|
|
Some(Options {
|
|
clarify: false,
|
|
..Options::default()
|
|
}),
|
|
);
|
|
let Outcome::Verified(p) = silent else {
|
|
panic!("{}: the scripted patch did not verify: {silent:?}", c.id)
|
|
};
|
|
let unasked = p.ledger.iter().filter(|e| e.consequential).count();
|
|
assert!(
|
|
unasked >= ledger::ASK_AT,
|
|
"{}: the patch made only {unasked} consequential choices: {:?}",
|
|
c.id,
|
|
p.ledger
|
|
);
|
|
lines.push(format!(
|
|
" {:<28} asks about {} ({}); the silent patch chose {} consequential",
|
|
c.id,
|
|
q.open.len(),
|
|
q.open
|
|
.iter()
|
|
.map(|o| o.what.as_str())
|
|
.collect::<Vec<_>>()
|
|
.join(", "),
|
|
unasked
|
|
));
|
|
}
|
|
println!("clarifying questions:\n{}", lines.join("\n"));
|
|
}
|
|
|
|
/// Precise requests stay one answer: no request in the set that is not open or
|
|
/// underspecified leaves two consequential choices open. And what the open set leaves.
|
|
#[test]
|
|
fn precise_requests_are_not_opened_or_asked() {
|
|
let reg = builtin_registry();
|
|
let mut detected = 0;
|
|
let mut open = 0;
|
|
for c in eval::cases().unwrap() {
|
|
let base = eval::base_graph(&c).unwrap();
|
|
let n = ledger::open_choices(&c.request, &Said::new(&c.request), &base, reg);
|
|
if c.open {
|
|
open += 1;
|
|
if n.len() >= ledger::OPEN_AT {
|
|
detected += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if is_ask(&c) {
|
|
assert!(n.len() >= ledger::ASK_AT, "{}", c.id);
|
|
continue;
|
|
}
|
|
assert!(
|
|
n.len() < ledger::OPEN_AT,
|
|
"{}: {} open: {n:?}",
|
|
c.id,
|
|
n.len()
|
|
);
|
|
assert_eq!(variants::wanted(None, &c.request, &base), 1, "{}", c.id);
|
|
}
|
|
println!(
|
|
"of the {open} cases labelled open, {detected} are also found open by the ledger \
|
|
from their words alone"
|
|
);
|
|
}
|
|
|
|
/// Every ledger entry of every proposal the eval shows reopens as an alternative set with
|
|
/// one click: from its own alternatives, written into the edit and each verified, the
|
|
/// value it chose first (no model asked); or, when none of those verifies, as a request in
|
|
/// the alternatives mode naming the edit and the choice, which an open session takes.
|
|
#[test]
|
|
fn any_ledger_entry_reopens_as_an_alternative_set() {
|
|
let reg = builtin_registry();
|
|
let (mut tried, mut local, mut shown, mut via_agent) = (0usize, 0usize, 0usize, 0usize);
|
|
let mut kinds: std::collections::BTreeMap<&str, (usize, usize)> = Default::default();
|
|
for c in eval::cases().unwrap() {
|
|
let base = eval::base_graph(&c).unwrap();
|
|
let (o, _) = replayed(&c, None);
|
|
for p in o.proposals() {
|
|
for (i, e) in p.ledger.iter().enumerate() {
|
|
tried += 1;
|
|
let kind = match &e.key {
|
|
None => "widget",
|
|
Some(k) if ledger::PRESENTATION.contains(&k.as_str()) => "presentation",
|
|
Some(_) if e.default => "default",
|
|
Some(_) => "value",
|
|
};
|
|
kinds.entry(kind).or_default().0 += 1;
|
|
match ledger::reopen(&base, p, i, reg, &InBrowser) {
|
|
Ok(set) => {
|
|
local += 1;
|
|
kinds.entry(kind).or_default().1 += 1;
|
|
shown += set.len();
|
|
assert!(set.len() >= 2 && set.len() <= variants::MAX_IN_VIEW);
|
|
assert_eq!(
|
|
dsl::semantics(&set[0].graph),
|
|
dsl::semantics(&p.graph),
|
|
"{}: the choice made comes first",
|
|
c.id
|
|
);
|
|
for a in &set {
|
|
assert!(a.verification.verified(), "{}", c.id);
|
|
let d = &a.variant.as_ref().unwrap().dimension;
|
|
assert!(!d.derived, "{}: {}", c.id, d.label());
|
|
assert_eq!(d.handle, e.handle, "{}", c.id);
|
|
let node = &a.graph.nodes[&e.node().unwrap()];
|
|
let got = match &e.key {
|
|
Some(k) => node.properties.get(k).cloned().unwrap_or_else(|| {
|
|
reg.get(&node.type_id)
|
|
.and_then(|w| {
|
|
w.parameters().into_iter().find(|p| p.name == *k)
|
|
})
|
|
.map(|p| p.default)
|
|
.unwrap_or_default()
|
|
}),
|
|
None => node.type_id.clone(),
|
|
};
|
|
assert!(
|
|
ledger::same_value(&got, &d.choice),
|
|
"{}: {} shows {got}",
|
|
c.id,
|
|
d.label()
|
|
);
|
|
assert!(eval::ledger_guard(a, &p.asked).0.is_empty(), "{}", c.id);
|
|
}
|
|
}
|
|
Err(_) => {
|
|
let ask = ledger::reopen_request(p, i).unwrap();
|
|
assert!(ask.contains(&e.label()) && ask.contains(p.patch.trim()));
|
|
let s = Session::new(
|
|
base.clone(),
|
|
reg,
|
|
&ask,
|
|
verify::Expect::default(),
|
|
Options {
|
|
alternatives: variants::DEFAULT_K,
|
|
..Options::default()
|
|
},
|
|
);
|
|
assert_eq!(s.alternatives(), variants::DEFAULT_K);
|
|
assert!(matches!(s.next(), Turn::Ask), "{}: {ask}", c.id);
|
|
via_agent += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
println!(
|
|
"reopened {local} of {tried} ledger entries from their own alternatives ({shown} \
|
|
verified alternatives shown, no model asked); the other {via_agent} reopen through \
|
|
the agent's alternatives mode\n by kind (entries, reopened locally): {kinds:?}"
|
|
);
|
|
assert!(tried >= 40);
|
|
assert_eq!(local + via_agent, tried);
|
|
assert!(
|
|
local * 2 >= tried,
|
|
"most entries reopen without a model: {local} of {tried}"
|
|
);
|
|
}
|
|
|
|
/// The lines between a node's markers are its code: for every corpus graph the plain
|
|
/// program is exactly what codegen emits, every node that binds an output has a span, the
|
|
/// spans nest or stand apart, and every output's binding — found where the tier check's
|
|
/// probes say the compiled program binds it — lies inside its node's span.
|
|
#[test]
|
|
fn explanations_anchor_to_the_right_rust_lines_for_every_corpus_graph() {
|
|
let reg = builtin_registry();
|
|
let mut anchored = 0usize;
|
|
let mut bindings = 0usize;
|
|
let mut documented = 0usize;
|
|
for e in corpus::entries() {
|
|
let g: Graph = e.parse().unwrap().graph;
|
|
let (plain, spans) = rust::primitives_to_rust_spans(&g, 1).unwrap();
|
|
assert_eq!(
|
|
plain,
|
|
rust::primitives_to_rust_steps(&g, 1).unwrap(),
|
|
"{}",
|
|
e.slug
|
|
);
|
|
let lines: Vec<&str> = plain.lines().collect();
|
|
for (i, a) in spans.iter().enumerate() {
|
|
assert!(a.first >= 1 && a.last <= lines.len() && a.first <= a.last);
|
|
for b in &spans[i + 1..] {
|
|
let apart = a.last < b.first || b.last < a.first;
|
|
let nested = (a.first <= b.first && b.last <= a.last)
|
|
|| (b.first <= a.first && a.last <= b.last);
|
|
assert!(apart || nested, "{}: {a:?} and {b:?} overlap", e.slug);
|
|
}
|
|
}
|
|
// Where the compiled program binds each output: the tier check's probe for it is
|
|
// written right after the binding, so the plain lines before the probe end with
|
|
// the node's code. That last line must be inside the node's span.
|
|
let mut probes: Vec<(NodeId, String)> = Vec::new();
|
|
for n in g.nodes.values() {
|
|
if let Some(t) = dsl::print::port_types(reg, n) {
|
|
for (port, _) in &t.outputs {
|
|
probes.push((n.id, port.clone()));
|
|
}
|
|
}
|
|
}
|
|
let probed = rust::primitives_to_rust_probed(&g, &probes).unwrap();
|
|
let mut before = 0usize;
|
|
for l in probed.lines() {
|
|
let Some(rest) = l.trim().strip_prefix("eprintln!(\"\\u{1f}") else {
|
|
before += 1;
|
|
continue;
|
|
};
|
|
let node = NodeId(rest.split_once('.').unwrap().0.parse().unwrap());
|
|
let span = spans
|
|
.iter()
|
|
.filter(|s| s.node == node)
|
|
.max_by_key(|s| s.depth)
|
|
.unwrap_or_else(|| panic!("{}: node {} has no span", e.slug, node.0));
|
|
assert!(
|
|
span.first <= before && before <= span.last,
|
|
"{}: node {}'s code ends at line {before}, outside its span {}..={}",
|
|
e.slug,
|
|
node.0,
|
|
span.first,
|
|
span.last
|
|
);
|
|
bindings += 1;
|
|
}
|
|
assert_eq!(
|
|
before,
|
|
lines.len(),
|
|
"{}: the probed program is the plain one",
|
|
e.slug
|
|
);
|
|
let ex = flowrs_core::graph::explain::explain(&g, &spans);
|
|
documented += ex.iter().filter(|x| x.doc.is_some()).count();
|
|
assert_eq!(ex.len(), spans.len());
|
|
anchored += 1;
|
|
}
|
|
println!(
|
|
"explanations anchored for {anchored} corpus graphs; {bindings} output bindings each \
|
|
inside its node's lines; {documented} explanations carry the widget's own description"
|
|
);
|
|
assert_eq!(anchored, flowrs_core::graph::corpus::entries().len());
|
|
assert!(anchored >= 20);
|
|
assert!(bindings >= 40);
|
|
}
|
|
|
|
/// An edit's rationale is anchored too: the Code view of a proposal shows each changed
|
|
/// node's reason beside its lines.
|
|
#[test]
|
|
fn a_proposals_rationale_sits_beside_its_nodes_lines() {
|
|
let c = eval::cases()
|
|
.unwrap()
|
|
.into_iter()
|
|
.find(|c| c.id == "gain-axis-repaired")
|
|
.unwrap();
|
|
let (o, _) = replayed(&c, None);
|
|
let Outcome::Verified(p) = o else { panic!() };
|
|
let (program, ex) = flowrs_core::graph::explain::explained(&p.graph, 1).unwrap();
|
|
let gain = p
|
|
.graph
|
|
.nodes
|
|
.values()
|
|
.find(|n| n.type_id == "dsp.gain")
|
|
.map(|n| n.id)
|
|
.unwrap();
|
|
let at = ex
|
|
.iter()
|
|
.find(|x| x.node == gain.0)
|
|
.expect("the gain has lines");
|
|
let code = program.lines().collect::<Vec<_>>()[at.first - 1..at.last].join("\n");
|
|
assert!(code.contains("gain") || code.contains("db"), "{code}");
|
|
let why: Vec<&str> = p
|
|
.rationale
|
|
.iter()
|
|
.filter(|r| dsl::handle_id(&r.handle) == Some(gain))
|
|
.map(|r| r.text.as_str())
|
|
.collect();
|
|
assert!(why.iter().all(|w| w.contains("+6 dB gain")), "{why:?}");
|
|
assert_eq!(edit::key_of(&eval::base_graph(&c).unwrap()), p.base_key);
|
|
}
|