399 lines
14 KiB
Rust
399 lines
14 KiB
Rust
//! An open request's alternatives through the real loop: one generation call, each
|
|
//! alternative checked and repaired on its own against the original graph, duplicates
|
|
//! merged, failures dropped, labels that name a handle each alternative changed.
|
|
//!
|
|
//! Browser checks (no compiler) keep these fast; `tests/agent_edit_eval.rs` runs the open
|
|
//! eval cases with cargo.
|
|
|
|
use flowrs_core::graph::corpus;
|
|
use flowrs_core::graph::edit::{self, Options, Outcome, Role, Session, Turn, variants};
|
|
use flowrs_core::graph::ir::Graph;
|
|
use flowrs_core::graph::verify::{Expect, InBrowser};
|
|
use flowrs_core::graph::widget::builtin_registry;
|
|
|
|
fn base(slug: &str) -> Graph {
|
|
corpus::get(slug).unwrap().parse().unwrap().graph
|
|
}
|
|
|
|
fn open(slug: &str, request: &str, k: usize) -> Session<'static> {
|
|
Session::new(
|
|
base(slug),
|
|
builtin_registry(),
|
|
request,
|
|
Expect::default(),
|
|
Options {
|
|
alternatives: k,
|
|
..Options::default()
|
|
},
|
|
)
|
|
}
|
|
|
|
fn alt(n: usize, handle: &str, patch: &str) -> String {
|
|
format!(
|
|
"ALTERNATIVE {n} · varies: choice {n} · handle: {handle} · probability: 0.{n}\n\
|
|
Alternative {n}.\n```patch\n{patch}\n```\n"
|
|
)
|
|
}
|
|
|
|
fn done(t: Turn) -> Outcome {
|
|
match t {
|
|
Turn::Done(o) => *o,
|
|
Turn::Ask => panic!("the loop asked for another round"),
|
|
}
|
|
}
|
|
|
|
/// The single-answer loop's prompt is what it was; an open request's replaces only the
|
|
/// one-answer instruction and says so in the first message.
|
|
#[test]
|
|
fn the_single_answer_prompt_is_unchanged_and_an_open_one_asks_for_k() {
|
|
let one = open("float_sum", "multiply them", 1);
|
|
assert_eq!(one.alternatives(), 1);
|
|
assert!(one.system().contains("then exactly one fenced"));
|
|
assert!(!one.system().contains("ALTERNATIVE"));
|
|
assert!(!one.messages()[0].content.contains("This request is open"));
|
|
assert!(one.system().contains(&edit::rules(1)));
|
|
|
|
let three = open("float_sum", "multiply them", 3);
|
|
assert_eq!(three.alternatives(), 3);
|
|
assert!(three.system().contains("Reply with 3 ALTERNATIVES"));
|
|
assert!(three.system().contains("probability"));
|
|
assert!(!three.system().contains("then exactly one fenced"));
|
|
assert!(
|
|
three.messages()[0]
|
|
.content
|
|
.contains("This request is open: offer 3 alternatives")
|
|
);
|
|
// Only the instruction differs: the language, the examples' format and the widgets
|
|
// are the same prompt.
|
|
assert!(three.system().contains("THE GRAPH LANGUAGE"));
|
|
assert!(three.system().contains("WIDGETS THAT MATTER HERE"));
|
|
// Never more than four.
|
|
assert_eq!(
|
|
open("float_sum", "x", 9).alternatives(),
|
|
variants::MAX_IN_VIEW
|
|
);
|
|
}
|
|
|
|
/// A failing alternative is repaired in its own thread: the generation reply and one
|
|
/// repair request naming it, nothing about the others. The verified ones wait.
|
|
#[test]
|
|
fn a_failing_alternative_is_repaired_on_its_own() {
|
|
let mut s = open("float_sum", "change what it computes", 3);
|
|
let reply = format!(
|
|
"Three edits.\n\n{}{}{}",
|
|
alt(1, "n1.value", "set n1 { value: 5 };"),
|
|
alt(2, "n2", "replace n2 = math.mul(a: n0.value, b: n1.value);"),
|
|
alt(3, "n0.value", "set n0 { value: 7 };"),
|
|
);
|
|
assert!(matches!(s.submit(&reply, &InBrowser), Turn::Ask));
|
|
let m = s.messages();
|
|
assert_eq!(
|
|
m.len(),
|
|
3,
|
|
"first message, the generation reply, one repair"
|
|
);
|
|
assert_eq!(m[1].role, Role::Assistant);
|
|
assert_eq!(m[2].role, Role::User);
|
|
assert!(
|
|
m[2].content
|
|
.starts_with("Round 2 of 3 for alternative 2 of 3 (\"choice 2\")"),
|
|
"{}",
|
|
m[2].content
|
|
);
|
|
assert!(
|
|
m[2].content
|
|
.contains("for alternative 2 only, against the original graph"),
|
|
"{}",
|
|
m[2].content
|
|
);
|
|
let o = done(s.submit(
|
|
"```patch\nreplace n2 = math.mul(a: n0.value, b: n1.value);\nn3.value = n2.product;\n```",
|
|
&InBrowser,
|
|
));
|
|
let Outcome::Alternatives(ps) = o else {
|
|
panic!("{o:?}")
|
|
};
|
|
assert_eq!(ps.len(), 3);
|
|
assert_eq!(
|
|
ps.iter().map(|p| p.rounds).collect::<Vec<_>>(),
|
|
vec![1, 2, 1]
|
|
);
|
|
assert_eq!(
|
|
s.rounds().iter().map(|r| r.alternative).collect::<Vec<_>>(),
|
|
vec![None, Some(2)]
|
|
);
|
|
let key = edit::key_of(&base("float_sum"));
|
|
for p in &ps {
|
|
let v = p.variant.as_ref().unwrap();
|
|
assert!(variants::names_changed_handle(&v.dimension, &p.changes));
|
|
assert!(p.verification.verified());
|
|
assert_eq!(p.base_key, key);
|
|
}
|
|
assert_eq!(ps[1].variant.as_ref().unwrap().dimension.handle, "n2");
|
|
assert_eq!(ps[2].variant.as_ref().unwrap().probability, Some(0.3));
|
|
// Each one applies to the person's graph on its own.
|
|
for p in &ps {
|
|
edit::apply_proposal(&base("float_sum"), p, builtin_registry()).unwrap();
|
|
}
|
|
}
|
|
|
|
/// Every repair starts from the generation reply: the second alternative's repair does not
|
|
/// carry the first one's thread.
|
|
#[test]
|
|
fn each_repair_starts_from_the_generation_reply() {
|
|
let mut s = open("float_sum", "change what it computes", 3);
|
|
let reply = format!(
|
|
"{}{}{}",
|
|
alt(1, "f", "let f = dsp.fft(signal: n9.value);"),
|
|
alt(2, "n2", "replace n2 = math.mul(a: n0.value, b: n1.value);"),
|
|
alt(3, "n0.value", "set n0 { value: 7 };"),
|
|
);
|
|
assert!(matches!(s.submit(&reply, &InBrowser), Turn::Ask));
|
|
assert!(s.messages()[2].content.contains("for alternative 1 of 3"));
|
|
// Alternative 1 repaired.
|
|
assert!(matches!(
|
|
s.submit("```patch\nset n1 { value: 5 };\n```", &InBrowser),
|
|
Turn::Ask
|
|
));
|
|
let m = s.messages();
|
|
assert_eq!(m.len(), 3, "{m:?}");
|
|
assert!(
|
|
m[2].content.contains("for alternative 2 of 3"),
|
|
"{}",
|
|
m[2].content
|
|
);
|
|
let o = done(s.submit(
|
|
"```patch\nreplace n2 = math.mul(a: n0.value, b: n1.value);\nn3.value = n2.product;\n```",
|
|
&InBrowser,
|
|
));
|
|
assert_eq!(o.proposals().len(), 3);
|
|
}
|
|
|
|
/// Two alternatives with the same result are one; one that never verifies is dropped,
|
|
/// and `fates` says what became of each.
|
|
#[test]
|
|
fn duplicates_merge_and_failures_drop_and_both_are_reported() {
|
|
let mut s = open("float_sum", "change what it computes", 3);
|
|
let bad = "let f = dsp.fft(signal: n9.value);";
|
|
let reply = format!(
|
|
"{}{}{}",
|
|
alt(1, "n1.value", "set n1 { value: 5 };"),
|
|
alt(2, "n1", "set n1 { value: 5 }"),
|
|
alt(3, "f", bad),
|
|
);
|
|
assert!(matches!(s.submit(&reply, &InBrowser), Turn::Ask));
|
|
assert!(matches!(
|
|
s.submit(&format!("```patch\n{bad}\n```"), &InBrowser),
|
|
Turn::Ask
|
|
));
|
|
let o = done(s.submit(&format!("```patch\n{bad}\n```"), &InBrowser));
|
|
let Outcome::Alternatives(ps) = o else {
|
|
panic!("{o:?}")
|
|
};
|
|
assert_eq!(ps.len(), 1, "one survivor is still shown, as one");
|
|
let fates: Vec<(usize, String)> = s.fates().into_iter().map(|f| (f.offered, f.fate)).collect();
|
|
assert_eq!(
|
|
fates,
|
|
vec![
|
|
(1, "shown".to_string()),
|
|
(2, "duplicate".to_string()),
|
|
(3, "failed".to_string())
|
|
]
|
|
);
|
|
assert!(s.fates()[2].detail.contains("not verified after 3 rounds"));
|
|
}
|
|
|
|
/// None verified: an honest failure. Prose and no patch: a question answered. One bare
|
|
/// patch with no headers: one alternative.
|
|
#[test]
|
|
fn the_other_ends_of_an_open_request() {
|
|
let bad = "let f = dsp.fft(signal: n9.value);";
|
|
let mut s = Session::new(
|
|
base("float_sum"),
|
|
builtin_registry(),
|
|
"x",
|
|
Expect::default(),
|
|
Options {
|
|
alternatives: 2,
|
|
max_rounds: 1,
|
|
..Options::default()
|
|
},
|
|
);
|
|
let o = done(s.submit(
|
|
&format!("{}{}", alt(1, "f", bad), alt(2, "f", bad)),
|
|
&InBrowser,
|
|
));
|
|
let Outcome::Failed(f) = o else {
|
|
panic!("{o:?}")
|
|
};
|
|
assert!(
|
|
f.reason.starts_with("none of the 2 alternatives verified"),
|
|
"{}",
|
|
f.reason
|
|
);
|
|
assert_eq!(f.patch.as_deref(), Some(bad));
|
|
|
|
let mut s = open("float_sum", "what does math.add do?", 3);
|
|
assert!(matches!(
|
|
done(s.submit("It adds its two inputs.", &InBrowser)),
|
|
Outcome::Answered { .. }
|
|
));
|
|
|
|
let mut s = open("float_sum", "x", 3);
|
|
let o = done(s.submit("Only one.\n```patch\nset n1 { value: 5 };\n```", &InBrowser));
|
|
let Outcome::Alternatives(ps) = o else {
|
|
panic!("{o:?}")
|
|
};
|
|
assert_eq!(ps.len(), 1);
|
|
assert!(ps[0].variant.as_ref().unwrap().dimension.derived);
|
|
}
|
|
|
|
/// More than k sent: the extra are not checked (a broken one is not repaired, which would
|
|
/// cost a round), and said so.
|
|
#[test]
|
|
fn alternatives_past_k_are_not_checked() {
|
|
let mut s = open("float_sum", "x", 2);
|
|
let reply = format!(
|
|
"{}{}{}",
|
|
alt(1, "n1", "set n1 { value: 5 };"),
|
|
alt(2, "n0", "set n0 { value: 7 };"),
|
|
alt(3, "f", "let f = dsp.fft(signal: n9.value);"),
|
|
);
|
|
let Outcome::Alternatives(ps) = done(s.submit(&reply, &InBrowser)) else {
|
|
panic!()
|
|
};
|
|
assert_eq!(ps.len(), 2);
|
|
let f = s.fates();
|
|
assert_eq!(f[2].fate, "beyond_view");
|
|
assert!(f[2].detail.contains("more than the 2 asked for"));
|
|
}
|
|
|
|
/// A patch that applies and then fails a later check (here the wire types: a spectrum fed
|
|
/// to a time-domain filter) is not shown, however it was labelled.
|
|
#[test]
|
|
fn an_alternative_that_applies_but_fails_a_check_is_not_shown() {
|
|
let mut s = Session::new(
|
|
base("dsp_filter_spectrum"),
|
|
builtin_registry(),
|
|
"x",
|
|
Expect::default(),
|
|
Options {
|
|
alternatives: 2,
|
|
max_rounds: 1,
|
|
..Options::default()
|
|
},
|
|
);
|
|
let reply = format!(
|
|
"{}{}",
|
|
alt(1, "n1.order", "set n1 { order: 2 };"),
|
|
alt(2, "lp", "let lp = dsp.filter.lowpass(input: n2.magnitude);"),
|
|
);
|
|
let Outcome::Alternatives(ps) = done(s.submit(&reply, &InBrowser)) else {
|
|
panic!()
|
|
};
|
|
assert_eq!(ps.len(), 1);
|
|
assert!(ps.iter().all(|p| p.verification.verified()));
|
|
assert_eq!(s.fates()[1].fate, "failed");
|
|
assert!(s.fates()[1].detail.contains("types"), "{:?}", s.fates());
|
|
}
|
|
|
|
/// A spent budget mid-repair keeps what already verified.
|
|
#[test]
|
|
fn stopping_keeps_the_verified_alternatives() {
|
|
let mut s = open("float_sum", "x", 3);
|
|
let reply = format!(
|
|
"{}{}",
|
|
alt(1, "n1", "set n1 { value: 5 };"),
|
|
alt(2, "f", "let f = dsp.fft(signal: n9.value);"),
|
|
);
|
|
assert!(matches!(s.submit(&reply, &InBrowser), Turn::Ask));
|
|
let o = s.stop("its token budget is spent");
|
|
let Outcome::Alternatives(ps) = o else {
|
|
panic!("{o:?}")
|
|
};
|
|
assert_eq!(ps.len(), 1);
|
|
assert_eq!(s.fates()[1].detail, "its token budget is spent");
|
|
// And the single-answer loop stops as it did: a failure carrying the last attempt.
|
|
let mut one = open("float_sum", "x", 1);
|
|
one.submit(
|
|
"```patch\nlet f = dsp.fft(signal: n9.value);\n```",
|
|
&InBrowser,
|
|
);
|
|
let Outcome::Failed(f) = one.stop("spent") else {
|
|
panic!()
|
|
};
|
|
assert_eq!(f.reason, "spent");
|
|
assert!(f.verification.is_some());
|
|
}
|
|
|
|
/// The examples an open request is shown are chosen to differ: no two of them are the
|
|
/// same program, and the most relevant comes first.
|
|
#[test]
|
|
fn an_open_request_is_shown_varied_examples() {
|
|
let s = open(
|
|
"dsp_filter_spectrum",
|
|
"filter the signal before the spectrum",
|
|
3,
|
|
);
|
|
let ex = s.examples();
|
|
assert_eq!(ex[0].slug, "dsp_filter_spectrum");
|
|
for (i, a) in ex.iter().enumerate() {
|
|
for b in &ex[i + 1..] {
|
|
let (ga, gb) = (a.parse().unwrap().graph, b.parse().unwrap().graph);
|
|
assert!(flowrs_core::graph::diff::distance(&ga, &gb) > 0.0);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Diverse retrieval keeps the nearest entry first, offers only relevant entries, spreads
|
|
/// its picks further apart than the nearest-k ranking does over a set of requests, and
|
|
/// for some requests picks differently.
|
|
#[test]
|
|
fn retrieval_spreads_its_examples() {
|
|
let apart = |hits: &[(&'static corpus::Entry, f64)]| -> f64 {
|
|
let gs: Vec<Graph> = hits.iter().map(|(e, _)| e.parse().unwrap().graph).collect();
|
|
let (mut sum, mut n) = (0.0, 0usize);
|
|
for (i, a) in gs.iter().enumerate() {
|
|
for b in &gs[i + 1..] {
|
|
sum += flowrs_core::graph::diff::distance(a, b);
|
|
n += 1;
|
|
}
|
|
}
|
|
if n == 0 { 0.0 } else { sum / n as f64 }
|
|
};
|
|
let (mut d_sum, mut n_sum) = (0.0, 0.0);
|
|
let requests = [
|
|
"compare a reading against a limit",
|
|
"add two numbers and print the sum",
|
|
"filter the tone and find its pitch",
|
|
"threshold the image and find the blobs",
|
|
"scale the value and clamp it",
|
|
"print a message",
|
|
// Only one entry uses morphology: variety never brings in an irrelevant example.
|
|
"use morphology",
|
|
];
|
|
let mut differs = 0;
|
|
for r in requests {
|
|
let diverse = corpus::retrieve(r, None, 3);
|
|
let near = corpus::nearest(r, None, 3);
|
|
assert_eq!(diverse[0].0.slug, near[0].0.slug, "{r}");
|
|
assert_eq!(diverse.len(), near.len(), "{r}");
|
|
assert!(
|
|
diverse.iter().all(|(_, s)| *s > 0.0),
|
|
"{r}: an irrelevant example"
|
|
);
|
|
d_sum += apart(&diverse);
|
|
n_sum += apart(&near);
|
|
let a: Vec<&str> = diverse.iter().map(|(e, _)| e.slug).collect();
|
|
let b: Vec<&str> = near.iter().map(|(e, _)| e.slug).collect();
|
|
if a != b {
|
|
differs += 1;
|
|
}
|
|
}
|
|
println!("mean pairwise distance: diverse {d_sum:.3}, nearest {n_sum:.3}, {differs} differ");
|
|
assert!(differs >= 1, "diverse retrieval never chose differently");
|
|
assert!(
|
|
d_sum > n_sum,
|
|
"diverse {d_sum} is not further apart than nearest {n_sum}"
|
|
);
|
|
}
|