//! The ownership gate of work/provenance: 10,000 random interleavings of a person's edits //! and an agent's patches, and in none does a handle the person wrote change unless the //! patch names it and Review marks it. //! //! The oracle is the test's own. It does not ask `provenance` who wrote anything: it keeps //! its own record of each handle's last writer, from a plain reading of the graph before //! and after every step (a node's widget, each parameter's value, what feeds each input), //! and its own list of what each patch names, from the statements it wrote. The code under //! test then has to agree with it on every step: //! //! - a patch that changes a person's handle without naming it is refused, with the graph //! left as it was — and one that changes none is never refused; //! - an applied patch overrides exactly the person's handles it changed: each is in the //! list Review marks ([`edit::apply_proposal_marked`], what the IDE's review uses), each //! has a row of the diff to mark, and nothing else is listed; //! - after every step, each handle's stored author is the oracle's. //! //! The person edits the graph the way the IDE does, through //! [`provenance::edit_as`]; the agent's patches go through `apply_proposal_marked`, the //! one door every surface (the IDE review, MCP, WebMCP) applies a proposal by. One in ten //! refused patches also goes through the checker, which must refuse it at its `apply` //! stage. An import's handles are the person's work too, and a third of the runs start //! from an imported graph. use flowrs_core::graph::corpus; use flowrs_core::graph::diff; use flowrs_core::graph::dsl::{self, patch::ports_of}; use flowrs_core::graph::edit::{self, Proposal}; use flowrs_core::graph::ir::{Graph, NodeId, WireMode}; use flowrs_core::graph::provenance::{self, Agent, Author, Handle}; use flowrs_core::graph::verify::{self, Expect, InBrowser, Stage, Status, Verification}; use flowrs_core::graph::widget::{WidgetRegistry, builtin_registry}; use std::collections::{BTreeMap, BTreeSet}; const INTERLEAVINGS: usize = 10_000; /// SplitMix64: small, fast, and the same sequence on every machine. struct Rng(u64); impl Rng { fn next(&mut self) -> u64 { self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); let mut z = self.0; z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); z ^ (z >> 31) } fn below(&mut self, n: usize) -> usize { (self.next() % n.max(1) as u64) as usize } fn chance(&mut self, percent: u64) -> bool { self.next() % 100 < percent } fn pick(&mut self, v: &[T]) -> Option { (!v.is_empty()).then(|| v[self.below(v.len())].clone()) } } /// Who last wrote a handle, as the oracle records it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Who { Person, Import, Agent, } impl Who { fn protected(self) -> bool { matches!(self, Who::Person | Who::Import) } } /// The graph as the oracle reads it: every handle and its state. fn state(g: &Graph) -> BTreeMap { let mut s = BTreeMap::new(); for n in g.nodes.values() { s.insert(Handle::Node { node: n.id.0 }, n.type_id.clone()); for (k, v) in &n.properties { s.insert( Handle::Parameter { node: n.id.0, key: k.clone(), }, v.clone(), ); } } let mut feeds: BTreeMap<(u64, String), Vec> = BTreeMap::new(); for w in g.wires.values() { feeds .entry((w.to_node.0, w.to_port.clone())) .or_default() .push(format!( "{}.{} {:?} {}", w.from_node.0, w.from_port, w.mode, w.feedback )); } for ((node, port), mut v) in feeds { v.sort(); s.insert(Handle::Input { node, port }, v.join("|")); } s } /// Every handle whose state differs. fn changed(a: &BTreeMap, b: &BTreeMap) -> BTreeSet { a.keys() .chain(b.keys()) .filter(|h| a.get(*h) != b.get(*h)) .cloned() .collect() } /// After a step: changed handles on nodes that still exist are `who`'s; the handles of a /// node that is gone are forgotten (there is nothing left to protect). fn record(oracle: &mut BTreeMap, after: &Graph, delta: &BTreeSet, who: Who) { for h in delta { if after.nodes.contains_key(&h.node()) { oracle.insert(h.clone(), who); } else { oracle.remove(h); } } oracle.retain(|h, _| after.nodes.contains_key(&h.node())); } /// What a patch names, written down as the patch is generated. #[derive(Default)] struct Names { handles: BTreeSet, removed: BTreeSet, } impl Names { fn covers(&self, h: &Handle) -> bool { self.handles.contains(h) || matches!(h, Handle::Parameter { node, .. } | Handle::Input { node, .. } if self.removed.contains(node)) } } fn ids(g: &Graph) -> Vec { let mut v: Vec = g.nodes.keys().copied().collect(); v.sort(); v } /// A value an agent might set `key` of `type_id` to: one of its choices, or a number /// near its default. fn agent_value(rng: &mut Rng, reg: &WidgetRegistry, type_id: &str, key: &str, cur: &str) -> String { if let Some(p) = reg .get(type_id) .and_then(|w| w.parameters().into_iter().find(|p| p.name == key)) { if let Some(c) = rng.pick(&p.choices) { return c; } if let Ok(d) = p.default.parse::() { let v = d + rng.below(4) as f64; return if p.ty.contains('f') { format!("{v:.1}") } else { format!("{}", v as i64) }; } } match cur.parse::() { Ok(n) => (n + 1 + rng.below(5) as i64).to_string(), Err(_) => cur.to_string(), } } /// One random agent patch against `g`, and what it names. fn agent_patch( rng: &mut Rng, g: &Graph, reg: &WidgetRegistry, types: &[String], ) -> (String, Names) { let mut out = Vec::new(); let mut names = Names::default(); let nodes = ids(g); let n_stmts = 1 + rng.below(3); for _ in 0..n_stmts { let Some(x) = rng.pick(&nodes) else { break }; if names.removed.contains(&x.0) { continue; } let node = &g.nodes[&x]; let h = dsl::handle(x); match rng.below(8) { // set a value 0 | 1 => { let mut keys: Vec = node.properties.keys().cloned().collect(); if let Some(w) = reg.get(&node.type_id) { keys.extend(w.parameters().into_iter().map(|p| p.name)); } keys.sort(); keys.dedup(); if let Some(k) = rng.pick(&keys) { let cur = node.properties.get(&k).cloned().unwrap_or_default(); let v = agent_value(rng, reg, &node.type_id, &k, &cur); out.push(format!("set {h} {{ {k}: {v} }};")); names .handles .insert(Handle::Parameter { node: x.0, key: k }); } } // unset one 2 => { let mut keys: Vec = node.properties.keys().cloned().collect(); keys.sort(); if let Some(k) = rng.pick(&keys) { out.push(format!("unset {h}.{k};")); names .handles .insert(Handle::Parameter { node: x.0, key: k }); } } // remove it, naming some of what it feeds 3 => { out.push(format!("remove {h};")); names.handles.insert(Handle::Node { node: x.0 }); names.removed.insert(x.0); let mut fed: Vec<(NodeId, String)> = g .wires .values() .filter(|w| w.from_node == x && w.to_node != x) .map(|w| (w.to_node, w.to_port.clone())) .collect(); fed.sort(); fed.dedup(); for (m, p) in fed { if rng.chance(50) && !names.removed.contains(&m.0) { out.push(format!("{}.{p} = _;", dsl::handle(m))); names.handles.insert(Handle::Input { node: m.0, port: p }); } } } // rewire an input, or leave it open 4 | 5 => { let ins = ports_of(reg, &node.type_id, false).unwrap_or_default(); let Some(p) = rng.pick(&ins) else { continue }; let src = rng .pick(&nodes) .filter(|y| *y != x && !names.removed.contains(&y.0)); let text = match src.and_then(|y| { let outs = ports_of(reg, &g.nodes[&y].type_id, true).unwrap_or_default(); rng.pick(&outs).map(|o| format!("{}.{o}", dsl::handle(y))) }) { Some(s) if rng.chance(75) => s, _ => "_".to_string(), }; out.push(format!("{h}.{p} = {text};")); names.handles.insert(Handle::Input { node: x.0, port: p }); } // restate it with nothing, naming some of what that drops 6 => { let mut fed: Vec = g .wires .values() .filter(|w| w.to_node == x) .map(|w| w.to_port.clone()) .collect(); fed.sort(); fed.dedup(); let mut args = Vec::new(); for p in fed { if rng.chance(50) { args.push(format!("{p}: _")); names.handles.insert(Handle::Input { node: x.0, port: p }); } } let ty = match node.type_id.as_str() { "IntLiteral" if rng.chance(50) => "FloatLiteral", "FloatLiteral" if rng.chance(50) => "IntLiteral", t => t, }; // Each value: restated (with a new value), dropped on purpose, or dropped. let mut keys: Vec = node.properties.keys().cloned().collect(); keys.sort(); let (mut props, mut unsets) = (Vec::new(), Vec::new()); for k in keys { match rng.below(3) { 0 => { let cur = node.properties[&k].clone(); let v = agent_value(rng, reg, ty, &k, &cur); props.push(format!("{k}: {v}")); } 1 => unsets.push(format!("unset {h}.{k};")), _ => continue, } names .handles .insert(Handle::Parameter { node: x.0, key: k }); } out.push(format!( "replace {h} = {ty}({}) {{ {} }};", args.join(", "), props.join(", ") )); out.extend(unsets); names.handles.insert(Handle::Node { node: x.0 }); } // a new node fed by this one _ => { let outs = ports_of(reg, &node.type_id, true).unwrap_or_default(); let Some(ty) = rng.pick(types) else { continue }; let ins = ports_of(reg, &ty, false).unwrap_or_default(); let arg = match (rng.pick(&outs), rng.pick(&ins)) { (Some(o), Some(i)) => format!("{i}: {h}.{o}"), _ => String::new(), }; out.push(format!("let a{} = {ty}({arg});", rng.below(1000))); } } } (out.join("\n"), names) } /// One random edit by the person, as the IDE makes it. fn person_edit(rng: &mut Rng, g: &mut Graph, reg: &WidgetRegistry, types: &[String]) { let nodes = ids(g); let Some(x) = rng.pick(&nodes) else { return }; let kind = rng.below(8); provenance::edit_as(g, &Author::Human, |g| match kind { // set a value 0..=2 => { let node = g.get_node_mut(x).unwrap(); let mut keys: Vec = node.properties.keys().cloned().collect(); keys.push("label".into()); keys.sort(); let k = keys[rng.below(keys.len())].clone(); node.properties .insert(k, (rng.below(50) as i64 - 10).to_string()); } // clear one 3 => { let node = g.get_node_mut(x).unwrap(); let mut keys: Vec = node.properties.keys().cloned().collect(); keys.sort(); if let Some(k) = rng.pick(&keys) { node.properties.remove(&k); } } // draw a wire onto an input 4 => { let Some(y) = rng.pick(&nodes).filter(|y| *y != x) else { return; }; let outs = ports_of(reg, &g.nodes[&y].type_id, true).unwrap_or_default(); let ins = ports_of(reg, &g.nodes[&x].type_id, false).unwrap_or_default(); if let (Some(o), Some(i)) = (rng.pick(&outs), rng.pick(&ins)) { let mode = [WireMode::Move, WireMode::Borrow, WireMode::Copy][rng.below(3)]; let _ = g.connect_replacing(y, &o, x, &i, mode); } } // delete a wire 5 => { let mut ws: Vec<_> = g.wires.keys().copied().collect(); ws.sort(); if let Some(w) = rng.pick(&ws) { g.remove_wire(w); } } // delete the node 6 => { g.remove_node(x); } // add one, and move another (layout: no one's) _ => { if let Some(ty) = rng.pick(types) { let id = g.add_node(&ty, 10.0, 10.0); g.get_node_mut(id) .unwrap() .properties .insert("label".into(), "mine".into()); } if let Some(n) = g.get_node_mut(x) { n.position.0 += 40.0; } } }); } fn empty_verification() -> Verification { serde_json::from_value(serde_json::json!({ "stages": [], "findings": [] })).unwrap() } /// The oracle's author for a stored one. fn stored_is(a: &Author, who: Who, agent: &Author) -> bool { match who { Who::Person => *a == Author::Human, Who::Import => matches!(a, Author::Imported { .. }), Who::Agent => a == agent || matches!(a, Author::Agent { .. }), } } #[derive(Default, Debug)] struct Tally { interleavings: usize, person_edits: usize, patches: usize, did_not_apply: usize, applied: usize, refused: usize, overrides: usize, overridden_patches: usize, checked_refusals: usize, violations: usize, } #[test] fn ten_thousand_interleavings_never_change_a_persons_handle_unnamed_or_unmarked() { let reg = builtin_registry(); let bases: Vec = corpus::entries() .iter() .filter_map(|e| e.parse().ok().map(|p| p.graph)) .filter(|g| g.nodes.len() >= 2) .collect(); assert!(bases.len() >= 10, "{} corpus graphs", bases.len()); let mut types: Vec = bases .iter() .flat_map(|g| g.nodes.values().map(|n| n.type_id.clone())) .collect(); types.sort(); types.dedup(); let mut rng = Rng(0x0f10_5eed_2026_1022); let mut t = Tally::default(); let mut first_violation: Option = None; for run in 0..INTERLEAVINGS { let mut g = bases[rng.below(bases.len())].clone(); let mut oracle: BTreeMap = BTreeMap::new(); // Where the graph came from: unattributed, an earlier agent's, or an import. match run % 3 { 0 => {} 1 => provenance::label(&mut g, &Author::agent("test/earlier", "c0")), _ => { provenance::label( &mut g, &Author::Imported { dialect: "puredata".into(), }, ); for h in state(&g).into_keys() { oracle.insert(h, Who::Import); } } } let steps = 4 + rng.below(9); for step in 0..steps { let before = g.clone(); let s0 = state(&before); if rng.chance(50) { person_edit(&mut rng, &mut g, reg, &types); t.person_edits += 1; let delta = changed(&s0, &state(&g)); record(&mut oracle, &g, &delta, Who::Person); } else { t.patches += 1; let (patch, names) = agent_patch(&mut rng, &before, reg, &types); let Ok(applied) = dsl::apply_patch(&before, &patch, reg) else { t.did_not_apply += 1; continue; }; let delta = changed(&s0, &state(&applied.graph)); let protected: BTreeSet = delta .iter() .filter(|h| oracle.get(*h).is_some_and(|w| w.protected())) .cloned() .collect(); let unnamed: Vec<&Handle> = protected.iter().filter(|h| !names.covers(h)).collect(); let agent = Agent::new("test/model", &format!("r{run}s{step}")); let by = agent.author(""); let proposal = Proposal { patch: patch.clone(), explain: String::new(), graph: applied.graph.clone(), changes: applied.changes.clone(), verification: empty_verification(), base_key: edit::key_of(&before), rounds: 1, variant: None, ledger: Vec::new(), rationale: Vec::new(), asked: String::new(), agent, overrides: Vec::new(), app: None, app_base: String::new(), }; let mut bad = |why: String| { t.violations += 1; first_violation.get_or_insert_with(|| { format!("run {run} step {step}: {why}\npatch:\n{patch}") }); }; match edit::apply_proposal_marked(&g, &proposal, reg) { Err(why) => { t.refused += 1; if unnamed.is_empty() { bad(format!( "refused a patch that names every person's handle it changes: {why}" )); } // The checker refuses it too, at `apply`, naming the handle. if t.refused % 10 == 1 && !unnamed.is_empty() { t.checked_refusals += 1; let c = verify::check_patch( &before, &patch, reg, &InBrowser, &Expect::default(), ); let v = &c.verification; let at_apply = matches!(v.status(Stage::Apply), Some(Status::Failed(_))); let says = v .errors() .any(|f| unnamed.iter().any(|h| f.message.contains(&h.render()))); if !(at_apply && says) { bad(format!("the checker did not refuse it at apply: {v:?}")); } } // Nothing changed: `g` was never replaced. } Ok((after, overrides)) => { t.applied += 1; if !unnamed.is_empty() { bad(format!( "changed {unnamed:?}, which the person wrote, unnamed" )); } let marked: BTreeSet = overrides.iter().map(|o| o.handle.clone()).collect(); if marked != protected { bad(format!( "Review marks {marked:?}; the person's handles changed are {protected:?}" )); } let rows = diff::diff(&before, &after); for o in &overrides { if !rows.iter().any(|c| o.handle.is_about(c)) { bad(format!("{} is marked but no change is about it", o.at)); } } if state(&after) != state(&applied.graph) { bad("the applied graph is not the patch's".into()); } t.overrides += overrides.len(); t.overridden_patches += usize::from(!overrides.is_empty()); g = after; record(&mut oracle, &g, &delta, Who::Agent); // What changed is the agent's, as stored. for h in delta.iter().filter(|h| g.nodes.contains_key(&h.node())) { let a = provenance::author_of(&g, h); if a != by { bad(format!("{} is stored as {a:?}, not the agent", h.render())); } } } } } // After every step, every handle the oracle knows is stored as its author. for (h, who) in &oracle { let a = provenance::author_of(&g, h); if !stored_is(&a, *who, &Author::Unknown) { t.violations += 1; first_violation.get_or_insert_with(|| { format!( "run {run} step {step}: {} is stored as {a:?}; the oracle says {who:?}", h.render() ) }); } } } t.interleavings += 1; } eprintln!("provenance property: {t:#?}"); assert_eq!(t.interleavings, INTERLEAVINGS); assert_eq!(t.violations, 0, "{}", first_violation.unwrap_or_default()); // The run exercised the rule, not only its absence. assert!(t.refused >= 500, "{t:?}"); assert!(t.overridden_patches >= 500, "{t:?}"); assert!(t.applied >= 5_000, "{t:?}"); assert!(t.checked_refusals >= 50, "{t:?}"); } /// The edit loop holds the model to the rule like any other check: an unnamed change to a /// person's handle fails `apply` at the statement that made it, the model names it in its /// repair, and the verified proposal carries what it overrides and who proposed it. #[test] fn the_edit_loop_refuses_an_unnamed_override_and_takes_a_named_one() { use flowrs_core::graph::edit::{Options, Outcome, Session, Turn}; let reg = builtin_registry(); let mut base = corpus::get("float_sum").unwrap().parse().unwrap().graph; provenance::label(&mut base, &Author::Human); let agent = Agent::new("anthropic/claude-test", "conv-7"); let mut s = Session::new( base.clone(), reg, "use 1 instead of the second number", Expect::default(), Options { clarify: false, agent: agent.clone(), ..Options::default() }, ); let t = s.submit("```patch\nremove n1;\n```", &InBrowser); assert!(matches!(t, Turn::Ask), "{t:?}"); let repair = &s.messages().last().unwrap().content; assert!( repair.contains("`remove n1` also disconnects n2.b, which the person wired"), "{repair}" ); let Turn::Done(o) = s.submit( "```patch\nremove n1;\nlet one = FloatLiteral { value: 1 };\nn2.b = one.value;\n```", &InBrowser, ) else { panic!("not done") }; let Outcome::Verified(p) = *o else { panic!("{o:?}") }; let at: BTreeSet<&str> = p.overrides.iter().map(|o| o.at.as_str()).collect(); assert_eq!( at, BTreeSet::from(["n1", "n1.value", "n2.b"]), "{:?}", p.overrides ); assert_eq!(p.agent, agent); let (after, overrides) = edit::apply_proposal_marked(&base, &p, reg).unwrap(); assert_eq!(overrides, p.overrides); let by = Author::agent("anthropic/claude-test", "conv-7"); let b = Handle::Input { node: 2, port: "b".into(), }; assert_eq!(provenance::author_of(&after, &b), by); assert_eq!( after.nodes[&NodeId(0)].origin.params["value"], Author::Human ); assert_eq!( provenance::node_authors(&after, NodeId(2)) .unwrap() .author(), Author::Mixed ); // n0, n2, n3; n0.value; n0.value→n2.a and n2.sum→n3.value. assert_eq!(provenance::share(&after).human, 6); } /// The page's WebMCP tools are an agent's edits too: one that would change a person's /// handle unnamed is refused and changes nothing; a named one is stamped as the page agent's /// and says what it overrode. #[test] fn the_pages_agent_tools_hold_to_the_rule() { use flowrs_core::graph::agent; let mut g = corpus::get("float_sum").unwrap().parse().unwrap().graph; provenance::label(&mut g, &Author::Human); let before = flowrs_core::graph::document::save(&g); let r = agent::call( &mut g, "graph_remove_node", &serde_json::json!({ "node": 1 }), ); assert_eq!(r["ok"], false, "{r}"); assert!(r["error"].as_str().unwrap().contains("n2.b"), "{r}"); assert_eq!( flowrs_core::graph::document::save(&g), before, "nothing changed" ); let r = agent::call( &mut g, "graph_set_properties", &serde_json::json!({ "node": 0, "properties": { "value": "4" } }), ); assert_eq!(r["ok"], true, "{r}"); assert_eq!(r["overrides"][0]["at"], "n0.value"); let h = Handle::Parameter { node: 0, key: "value".into(), }; assert_eq!( provenance::author_of(&g, &h), Author::agent(provenance::UNKNOWN_MODEL, agent::WEBMCP_CONVERSATION) ); // A patch through the page: checked, refused unnamed, applied named and stamped. let r = agent::call( &mut g, "graph_patch_apply", &serde_json::json!({ "patch": "remove n1;" }), ); assert_eq!(r["ok"], false, "{r}"); let r = agent::call( &mut g, "graph_patch_apply", &serde_json::json!({ "patch": "let k = FloatLiteral { value: 4 };\nn2.b = k.value;" }), ); assert_eq!(r["applied"], true, "{r}"); assert_eq!(r["overrides"][0]["at"], "n2.b"); let b = Handle::Input { node: 2, port: "b".into(), }; assert!(matches!( provenance::author_of(&g, &b), Author::Agent { .. } )); }