366 lines
12 KiB
Rust
366 lines
12 KiB
Rust
//! The checker loop end to end: a patch against an existing graph, every stage, and
|
|
//! where each failure is reported.
|
|
|
|
use flowrs_core::graph::corpus;
|
|
use flowrs_core::graph::dsl;
|
|
use flowrs_core::graph::edit::{self, Options, Outcome, Session, Turn};
|
|
use flowrs_core::graph::ir::Graph;
|
|
use flowrs_core::graph::verify::{self, Expect, InBrowser, Native, Stage, Status};
|
|
use flowrs_core::graph::widget::builtin_registry;
|
|
|
|
fn base(slug: &str) -> Graph {
|
|
corpus::get(slug).unwrap().parse().unwrap().graph
|
|
}
|
|
|
|
fn have_cargo() -> bool {
|
|
let ok = std::process::Command::new("cargo")
|
|
.arg("-V")
|
|
.output()
|
|
.is_ok();
|
|
if !ok {
|
|
assert_eq!(
|
|
std::env::var("FLOWRS_ALLOW_MISSING_CARGO").as_deref(),
|
|
Ok("1")
|
|
);
|
|
}
|
|
ok
|
|
}
|
|
|
|
/// A wire whose named axes disagree is reported at the patch line that drew it, with the
|
|
/// ports of both ends as typed context.
|
|
#[test]
|
|
fn an_axis_mismatch_is_reported_at_its_line_with_typed_context() {
|
|
let c = verify::check_patch(
|
|
&base("dsp_filter_spectrum"),
|
|
"set n1 { order: 2 };\nlet lp = dsp.filter.lowpass(input: n2.magnitude);",
|
|
builtin_registry(),
|
|
&InBrowser,
|
|
&Expect::default(),
|
|
);
|
|
let v = &c.verification;
|
|
assert!(!v.verified());
|
|
assert_eq!(
|
|
v.status(Stage::Types),
|
|
Some(&Status::Failed("1 problem".into()))
|
|
);
|
|
assert_eq!(v.status(Stage::Compile), Some(&Status::NotRun));
|
|
let f = v.errors().next().unwrap();
|
|
assert_eq!(f.at, Some(dsl::Pos { line: 2, col: 36 }), "{f:?}");
|
|
assert!(
|
|
f.message.contains("n2.magnitude → lp (n3).input"),
|
|
"{}",
|
|
f.message
|
|
);
|
|
assert!(f.message.contains("freq"), "{}", f.message);
|
|
let hint = f.hint.as_deref().unwrap();
|
|
assert!(
|
|
hint.contains("n2 (dsp.fft) outputs: magnitude: Signal(f64)[freq:?]"),
|
|
"{hint}"
|
|
);
|
|
assert!(
|
|
hint.contains("lp (n3) (dsp.filter.lowpass) inputs: input: Signal(f64)[time:n]"),
|
|
"{hint}"
|
|
);
|
|
}
|
|
|
|
/// The browser has no compiler: compile and tiers are SKIPPED and say why; nothing claims
|
|
/// they passed.
|
|
#[test]
|
|
fn in_the_browser_compile_and_tiers_are_skipped_not_passed() {
|
|
let c = verify::check_patch(
|
|
&base("float_sum"),
|
|
"set n0 { value: 4 }",
|
|
builtin_registry(),
|
|
&InBrowser,
|
|
&Expect::from_json(&serde_json::json!({"n2.sum": 7})).unwrap(),
|
|
);
|
|
let v = c.verification;
|
|
assert!(v.verified(), "{}", v.feedback());
|
|
assert!(!v.complete());
|
|
assert!(
|
|
matches!(v.status(Stage::Compile), Some(Status::Skipped(why)) if why.contains("browser"))
|
|
);
|
|
assert!(
|
|
v.summary().contains("no Rust compiler runs in the browser"),
|
|
"{}",
|
|
v.summary()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_test_vector_that_fails_names_the_node_and_what_it_computed() {
|
|
let c = verify::check_patch(
|
|
&base("float_sum"),
|
|
"set n0 { value: 4 }",
|
|
builtin_registry(),
|
|
&InBrowser,
|
|
&Expect::from_json(&serde_json::json!({"n2.sum": 5})).unwrap(),
|
|
);
|
|
let v = c.verification;
|
|
assert!(!v.verified());
|
|
let f = v.errors().find(|f| f.stage == Stage::Tests).unwrap();
|
|
assert_eq!(f.name.as_deref(), Some("n2"));
|
|
assert!(
|
|
f.message
|
|
.contains("expected 5, the interpreter computed 7.0"),
|
|
"{}",
|
|
f.message
|
|
);
|
|
}
|
|
|
|
/// A compile error is mapped back to the node whose code rustc stopped on, and to the
|
|
/// patch line that wrote that node — given a toolchain the person told to trust code.
|
|
#[test]
|
|
fn a_compile_error_lands_on_the_node_that_caused_it() {
|
|
if !have_cargo() {
|
|
return;
|
|
}
|
|
struct Trusting(Native);
|
|
impl verify::Toolchain for Trusting {
|
|
fn describe(&self) -> String {
|
|
self.0.describe()
|
|
}
|
|
fn compile(&self, p: &str) -> flowrs_core::graph::toolchain::Compile {
|
|
self.0.compile(p)
|
|
}
|
|
fn tiers(
|
|
&self,
|
|
g: &Graph,
|
|
r: &flowrs_core::graph::widget::WidgetRegistry,
|
|
) -> flowrs_core::graph::toolchain::Tiers {
|
|
self.0.tiers(g, r)
|
|
}
|
|
fn code_trusted(&self, _: &Graph) -> Result<(), String> {
|
|
Ok(())
|
|
}
|
|
}
|
|
let c = verify::check_patch(
|
|
&base("float_sum"),
|
|
"remove n3;\nlet bad = Spawn { src: \"let x: i32 = \\\"no\\\";\" };",
|
|
builtin_registry(),
|
|
&Trusting(Native::default()),
|
|
&Expect::default(),
|
|
);
|
|
let v = c.verification;
|
|
assert!(
|
|
matches!(v.status(Stage::Compile), Some(Status::Failed(_))),
|
|
"{}",
|
|
v.feedback()
|
|
);
|
|
let f = v.errors().find(|f| f.stage == Stage::Compile).unwrap();
|
|
assert_eq!(f.name.as_deref(), Some("bad (n4)"), "{f:?}");
|
|
assert_eq!(f.at, Some(dsl::Pos { line: 2, col: 1 }));
|
|
// Untrusted, the same source is neither compiled nor run, and the report says so.
|
|
let c = verify::check_patch(
|
|
&base("float_sum"),
|
|
"remove n3;\nlet bad = Spawn { src: \"let x: i32 = \\\"no\\\";\" };",
|
|
builtin_registry(),
|
|
&Native::default(),
|
|
&Expect::default(),
|
|
);
|
|
assert!(matches!(
|
|
c.verification.status(Stage::Compile),
|
|
Some(Status::Skipped(why)) if why.contains("not compiled or run without the person's trust")
|
|
));
|
|
}
|
|
|
|
/// The full loop with a real compiler: a first reply that breaks the graph, its feedback,
|
|
/// a repaired reply, and a proposal that verified completely.
|
|
#[test]
|
|
fn the_loop_repairs_a_patch_and_ends_with_a_complete_proof() {
|
|
if !have_cargo() {
|
|
return;
|
|
}
|
|
let mut s = Session::new(
|
|
base("float_sum"),
|
|
builtin_registry(),
|
|
"multiply instead of adding",
|
|
Expect::from_json(&serde_json::json!({"n2.product": 6})).unwrap(),
|
|
Options::default(),
|
|
);
|
|
let tc = Native::default();
|
|
assert!(matches!(s.next(), Turn::Ask));
|
|
let t = s.submit(
|
|
"```patch\nreplace n2 = math.mul(a: n0.value, b: n1.value);\n```",
|
|
&tc,
|
|
);
|
|
assert!(matches!(t, Turn::Ask));
|
|
let feedback = &s.messages().last().unwrap().content;
|
|
assert!(feedback.contains("Round 1 of 3"), "{feedback}");
|
|
assert!(
|
|
feedback.contains("math.mul has no output `sum`"),
|
|
"{feedback}"
|
|
);
|
|
let t = s.submit(
|
|
"Multiplies.\n```patch\nreplace n2 = math.mul(a: n0.value, b: n1.value);\nn3.value = n2.product;\n```",
|
|
&tc,
|
|
);
|
|
let Turn::Done(outcome) = t else { panic!() };
|
|
let Outcome::Verified(p) = *outcome else {
|
|
panic!("{outcome:?}")
|
|
};
|
|
assert_eq!(p.rounds, 2);
|
|
assert!(p.verification.complete(), "{}", p.verification.feedback());
|
|
assert_eq!(p.explain, "Multiplies.");
|
|
assert!(
|
|
p.diff_lines()
|
|
.contains(&"~ node 2 type math.add -> math.mul".to_string())
|
|
);
|
|
// Applying it checks the base: the same graph applies, a changed one is refused.
|
|
let current = base("float_sum");
|
|
assert!(edit::apply_proposal(¤t, &p, builtin_registry()).is_ok());
|
|
let mut moved_on = current.clone();
|
|
moved_on
|
|
.nodes
|
|
.get_mut(&flowrs_core::NodeId(0))
|
|
.unwrap()
|
|
.properties
|
|
.insert("value".into(), "9".into());
|
|
assert!(
|
|
edit::apply_proposal(&moved_on, &p, builtin_registry())
|
|
.unwrap_err()
|
|
.contains("changed since this edit was proposed")
|
|
);
|
|
}
|
|
|
|
/// Out of rounds is an honest failure that carries the last attempt and its errors.
|
|
#[test]
|
|
fn running_out_of_rounds_is_a_failure_with_reasons() {
|
|
let mut s = Session::new(
|
|
base("hello"),
|
|
builtin_registry(),
|
|
"fft the message",
|
|
Expect::default(),
|
|
Options {
|
|
max_rounds: 2,
|
|
..Options::default()
|
|
},
|
|
);
|
|
s.submit(
|
|
"```patch\nlet f = dsp.fft(signal: n0.value);\n```",
|
|
&InBrowser,
|
|
);
|
|
let Turn::Done(o) = s.submit(
|
|
"```patch\nlet f = dsp.fft(signal: n0.value.clone());\n```",
|
|
&InBrowser,
|
|
) else {
|
|
panic!()
|
|
};
|
|
let Outcome::Failed(f) = *o else { panic!() };
|
|
assert_eq!(f.rounds, 2);
|
|
assert!(
|
|
f.reason.contains("not verified after 2 rounds"),
|
|
"{}",
|
|
f.reason
|
|
);
|
|
assert!(
|
|
f.verification
|
|
.unwrap()
|
|
.errors()
|
|
.any(|e| e.message.contains("cannot connect"))
|
|
);
|
|
}
|
|
|
|
/// The report travels: the server serializes it, the editor reads it back, and the two
|
|
/// must be the same report (a stage's status and detail, every finding's position).
|
|
#[test]
|
|
fn a_verification_survives_the_trip_to_the_editor() {
|
|
let c = verify::check_patch(
|
|
&base("float_sum"),
|
|
"set n0 { value: 4 };\nlet f = dsp.fft(signal: n9.x);",
|
|
builtin_registry(),
|
|
&InBrowser,
|
|
&Expect::default(),
|
|
);
|
|
let json = serde_json::to_string(&c.verification).unwrap();
|
|
let back: verify::Verification = serde_json::from_str(&json).unwrap();
|
|
assert_eq!(back, c.verification);
|
|
let ok = verify::check_patch(
|
|
&base("float_sum"),
|
|
"set n0 { value: 4 }",
|
|
builtin_registry(),
|
|
&InBrowser,
|
|
&Expect::default(),
|
|
);
|
|
let back: verify::Verification =
|
|
serde_json::from_value(serde_json::to_value(&ok.verification).unwrap()).unwrap();
|
|
assert_eq!(back, ok.verification);
|
|
assert!(back.verified());
|
|
}
|
|
|
|
/// A toolchain that climbs past the stages: the browser's, with one piece of evidence.
|
|
struct WithEvidence(verify::Evidence);
|
|
|
|
impl verify::Toolchain for WithEvidence {
|
|
fn describe(&self) -> String {
|
|
"the browser, and a twin".into()
|
|
}
|
|
fn compile(&self, program: &str) -> flowrs_core::graph::toolchain::Compile {
|
|
verify::Toolchain::compile(&InBrowser, program)
|
|
}
|
|
fn tiers(
|
|
&self,
|
|
graph: &flowrs_core::graph::ir::Graph,
|
|
registry: &flowrs_core::graph::widget::WidgetRegistry,
|
|
) -> flowrs_core::graph::toolchain::Tiers {
|
|
verify::Toolchain::tiers(&InBrowser, graph, registry)
|
|
}
|
|
fn evidence(&self, _graph: &flowrs_core::graph::ir::Graph) -> Vec<verify::Evidence> {
|
|
vec![self.0.clone()]
|
|
}
|
|
}
|
|
|
|
/// Evidence rides with the verification, a failure in it fails the verification and is
|
|
/// told to the model with each violation, and it survives the trip to the editor.
|
|
#[test]
|
|
fn failed_evidence_fails_the_verification_and_says_where() {
|
|
let detail = serde_json::json!({
|
|
"violations": [{
|
|
"class": "clock_disabled",
|
|
"register": "GPIOA.MODER",
|
|
"pc": 0x0800_0132u32,
|
|
"message": "GPIOA's clock is off",
|
|
}],
|
|
});
|
|
let failed = verify::Evidence {
|
|
rung: "twin_trace".into(),
|
|
status: Status::Failed("1 error".into()),
|
|
report: detail.clone(),
|
|
};
|
|
let c = verify::check_patch(
|
|
&base("float_sum"),
|
|
"set n0 { value: 4 }",
|
|
builtin_registry(),
|
|
&WithEvidence(failed),
|
|
&Expect::default(),
|
|
);
|
|
let v = &c.verification;
|
|
assert_eq!(v.evidence.len(), 1);
|
|
assert!(!v.verified());
|
|
assert!(v.summary().contains("twin_trace failed"), "{}", v.summary());
|
|
let fb = v.feedback();
|
|
assert!(
|
|
fb.contains("[clock_disabled] GPIOA.MODER at pc 0x08000132"),
|
|
"{fb}"
|
|
);
|
|
let back: verify::Verification =
|
|
serde_json::from_value(serde_json::to_value(v).unwrap()).unwrap();
|
|
assert_eq!(&back, v);
|
|
assert_eq!(v.to_json()["evidence"][0]["rung"], "twin_trace");
|
|
|
|
// Passed evidence leaves a verified graph verified, and says what it found.
|
|
let passed = verify::Evidence {
|
|
rung: "twin_trace".into(),
|
|
status: Status::Passed("no violations".into()),
|
|
report: detail,
|
|
};
|
|
let c = verify::check_patch(
|
|
&base("float_sum"),
|
|
"set n0 { value: 4 }",
|
|
builtin_registry(),
|
|
&WithEvidence(passed),
|
|
&Expect::default(),
|
|
);
|
|
assert!(c.verification.verified());
|
|
assert!(c.verification.feedback().contains("twin_trace: passed"));
|
|
}
|