flowrs-engine/crates/flowrs-core/tests/agent_edit_eval.rs

//! The agent edit eval set, replayed from its recorded replies.
//!
//! No model and no key: each case's replies are replayed round by round through the real
//! loop — the real parser, applier, checker and `cargo check` — and the result is graded
//! against the case's properties. The pass rate is printed; every case must pass, because
//! the recorded replies are fixed and a regression in the loop is the only thing that can
//! change an outcome.

use flowrs_core::graph::edit::{eval, variants};
use flowrs_core::graph::verify::Native;
use flowrs_core::graph::widget::builtin_registry;

#[test]
fn the_recorded_eval_set_passes() {
    if std::process::Command::new("cargo")
        .arg("-V")
        .output()
        .is_err()
    {
        assert_eq!(
            std::env::var("FLOWRS_ALLOW_MISSING_CARGO").as_deref(),
            Ok("1"),
            "no cargo on PATH"
        );
        return;
    }
    let cases = eval::cases().expect("cases.json parses");
    assert!(cases.len() >= 12, "{}", cases.len());
    let tc = Native::default();
    let results: Vec<_> = cases
        .iter()
        .map(|c| eval::replay(c, builtin_registry(), &tc))
        .collect();
    let report = eval::report(&results);
    println!("{report}");
    assert!(results.iter().all(|r| r.passed), "\n{report}");
    // The set exercises the repair loop and an honest failure, not only first-try passes.
    assert!(
        results
            .iter()
            .filter(|r| r.rounds >= 2 && r.outcome == "verified")
            .count()
            >= 4
    );
    assert!(results.iter().any(|r| r.outcome == "failed"));
    assert!(results.iter().any(|r| r.outcome == "answered"));
}

/// Branch row 1's acceptance (work/variants), offline, on scripted replies:
///
/// - quality: every case that is not open passes in the single-answer loop (k = 1); every
///   alternative shown at k = 3 is verified (the grader's `alternatives_guard`); at least
///   10 of the 12 open cases show two or more verified alternatives;
/// - diversity: the alternatives' mean pairwise edit distance is at least 1.5 times that of
///   three independent default-prompt samples of the same request; no two shown share a
///   `semantic_key`; at most one in five is within 0.1 of a retrieved example; every label
///   names a handle its alternative changed.
///
/// The scripted default samples are written the way default sampling behaves (mode
/// collapse: the same answer, or one parameter apart). The figure on a real model comes
/// from `scripts/agent-eval.sh --endpoint … --record`, which records both.
#[test]
fn open_cases_meet_the_variants_acceptance() {
    if std::process::Command::new("cargo")
        .arg("-V")
        .output()
        .is_err()
    {
        assert_eq!(
            std::env::var("FLOWRS_ALLOW_MISSING_CARGO").as_deref(),
            Ok("1"),
            "no cargo on PATH"
        );
        return;
    }
    let cases = eval::cases().unwrap();
    let tc = Native::default();
    let (open, single): (Vec<_>, Vec<_>) = cases.iter().partition(|c| c.open);
    assert_eq!(open.len(), 12, "the open-ended set");
    assert!(single.len() >= 19);

    // Quality at k = 1.
    for c in &single {
        let r = eval::replay(c, builtin_registry(), &tc);
        assert!(r.passed && r.outcome != "alternatives", "{}: {r:?}", c.id);
    }

    let mut two_or_more = 0usize;
    let mut shown = 0usize;
    let mut near = 0usize;
    let (mut alt_spread, mut base_spread, mut measured) = (0.0f64, 0.0f64, 0usize);
    let (mut alt_calls, mut base_calls) = (0usize, 0usize);
    let mut lines = Vec::new();
    for c in &open {
        let r = eval::replay(c, builtin_registry(), &tc);
        assert!(r.passed, "{}: {:?}", c.id, r.failures);
        assert_eq!(r.outcome, "alternatives", "{}", c.id);
        // Re-derive the guard here rather than trust the grade: every one verified, no two
        // sharing a semantic key, each label naming a handle it changed.
        let mut s = eval::session(c, builtin_registry()).unwrap();
        for rec in &c.recorded {
            s.submit(&rec.reply, &tc);
        }
        let flowrs_core::graph::edit::Turn::Done(o) = s.next() else {
            panic!("{}: the loop wanted more replies", c.id)
        };
        let flowrs_core::graph::edit::Outcome::Alternatives(ps) = *o else {
            panic!("{}: not alternatives", c.id)
        };
        let mut keys = std::collections::HashSet::new();
        for p in &ps {
            assert!(p.verification.verified(), "{}", c.id);
            assert!(
                keys.insert(p.graph.semantic_key()),
                "{}: a repeated key",
                c.id
            );
            let v = p.variant.as_ref().unwrap();
            assert!(
                variants::names_changed_handle(&v.dimension, &p.changes),
                "{}: {}",
                c.id,
                v.dimension.label()
            );
        }
        assert!(ps.len() <= variants::MAX_IN_VIEW);
        assert_eq!(ps.len(), r.alternatives);

        if r.alternatives >= 2 {
            two_or_more += 1;
        }
        shown += r.alternatives;
        near += r
            .example_distances
            .iter()
            .filter(|d| **d <= variants::NEAR_COPY)
            .count();
        alt_calls += r.rounds;
        let b = eval::baseline(c, builtin_registry(), &tc).expect("every open case has a baseline");
        assert_eq!(b.verified, 3, "{}: a baseline run did not verify", c.id);
        base_calls += b.calls;
        if r.alternatives >= 2 {
            alt_spread += r.spread;
            base_spread += b.spread;
            measured += 1;
        }
        lines.push(format!(
            "  {:<34} {} shown · spread {:>5.2} vs default {:>5.2} · calls {} vs {} · nearest example {:?}",
            c.id,
            r.alternatives,
            r.spread,
            b.spread,
            r.rounds,
            b.calls,
            r.example_distances
                .iter()
                .map(|d| (d * 100.0).round() / 100.0)
                .collect::<Vec<_>>()
        ));
    }
    let alt_mean = alt_spread / measured as f64;
    let base_mean = base_spread / measured as f64;
    println!(
        "open cases: {two_or_more} of {} show two or more verified alternatives; {shown} shown; \
         {near} within {} of a retrieved example\n\
         spread: alternatives {alt_mean:.2}, three default samples {base_mean:.2} ({:.2}x over {measured} cases)\n\
         model calls: alternatives {alt_calls}, default samples {base_calls}\n{}",
        open.len(),
        variants::NEAR_COPY,
        alt_mean / base_mean,
        lines.join("\n")
    );
    assert!(
        two_or_more >= 10,
        "{two_or_more} of 12 open cases show two or more"
    );
    assert!(
        alt_mean >= 1.5 * base_mean,
        "alternatives {alt_mean:.2} apart, default samples {base_mean:.2}: under 1.5x"
    );
    assert!(
        near * 5 <= shown,
        "{near} of {shown} alternatives are near copies"
    );
}

/// The set properties fail a set that misses them: too few alternatives, too close
/// together, or near copies of the examples.
#[test]
fn the_set_properties_fail_a_set_that_misses_them() {
    let tc = flowrs_core::graph::verify::InBrowser;
    let mut c = eval::cases()
        .unwrap()
        .into_iter()
        .find(|c| c.id == "open-clean-temperature-log")
        .unwrap();
    let r = eval::replay(&c, builtin_registry(), &tc);
    assert!(r.passed, "{r:?}");
    assert_eq!(r.alternatives, 2);
    c.properties = vec![
        eval::Property::DistinctAlternatives(3),
        eval::Property::DiversityAtLeast(r.spread + 1.0),
        eval::Property::NotCopyOfExample(1.0),
    ];
    let r = eval::replay(&c, builtin_registry(), &tc);
    assert!(!r.passed);
    let has = |s: &str| r.failures.iter().any(|f| f.contains(s));
    assert!(has("2 distinct alternative(s) shown, at least 3"), "{r:?}");
    assert!(has("apart on average"), "{r:?}");
    assert!(has("near copy of an example"), "{r:?}");
    // A single answer has no set to measure.
    let mut one = eval::cases()
        .unwrap()
        .into_iter()
        .find(|c| c.id == "iir-to-highpass")
        .unwrap();
    one.properties = vec![eval::Property::DistinctAlternatives(2)];
    assert!(!eval::replay(&one, builtin_registry(), &tc).passed);
}

/// The grader is not a rubber stamp on alternatives either: a set with a repeat, an
/// unverified member or a label naming an untouched handle fails the guard.
#[test]
fn the_alternatives_guard_refuses_what_it_should() {
    let c = eval::cases()
        .unwrap()
        .into_iter()
        .find(|c| c.id == "open-filter-less-ringing")
        .unwrap();
    let tc = flowrs_core::graph::verify::InBrowser;
    let mut s = eval::session(&c, builtin_registry()).unwrap();
    for rec in &c.recorded {
        s.submit(&rec.reply, &tc);
    }
    let flowrs_core::graph::edit::Turn::Done(o) = s.next() else {
        panic!()
    };
    let flowrs_core::graph::edit::Outcome::Alternatives(ps) = *o else {
        panic!()
    };
    let base = eval::base_graph(&c).unwrap();
    assert!(eval::alternatives_guard(&ps, &base).is_empty());

    let mut repeat = ps.clone();
    repeat.push(ps[0].clone());
    assert!(
        eval::alternatives_guard(&repeat, &base)
            .iter()
            .any(|f| f.contains("are the same edit")),
    );
    let mut unverified = ps.clone();
    unverified[1].verification.stages[0].status =
        flowrs_core::graph::verify::Status::Failed("x".into());
    assert!(
        eval::alternatives_guard(&unverified, &base)
            .iter()
            .any(|f| f.contains("not verified"))
    );
    let mut mislabelled = ps.clone();
    mislabelled[0].variant.as_mut().unwrap().dimension.handle = "n0".into();
    assert!(
        eval::alternatives_guard(&mislabelled, &base)
            .iter()
            .any(|f| f.contains("names no handle it changed"))
    );
    let mut swapped = ps.clone();
    swapped[0].graph = ps[1].graph.clone();
    assert!(
        eval::alternatives_guard(&swapped, &base)
            .iter()
            .any(|f| f.contains("does not re-apply"))
    );
    let mut five = ps.clone();
    five.extend(ps.iter().cloned());
    assert!(
        eval::alternatives_guard(&five, &base)
            .iter()
            .any(|f| f.contains("more than 4"))
    );
}

/// A reply that is wrong is graded wrong: the grader is not a rubber stamp. Every case's
/// LAST recorded reply is dropped, so a case that needed it either asks for a reply that
/// is not there or ends on its earlier, failing round — and must not pass.
#[test]
fn the_grader_fails_a_case_whose_fix_is_missing() {
    let cases = eval::cases().unwrap();
    let tc = flowrs_core::graph::verify::InBrowser;
    let mut graded = 0;
    for mut c in cases
        .into_iter()
        .filter(|c| c.recorded.len() >= 2 && c.outcome == "verified")
    {
        c.recorded.pop();
        let r = eval::replay(&c, builtin_registry(), &tc);
        assert!(!r.passed, "{} passed without its fix: {r:?}", c.id);
        graded += 1;
    }
    assert!(graded >= 4);
}

fn case(id: &str) -> eval::Case {
    eval::cases()
        .unwrap()
        .into_iter()
        .find(|c| c.id == id)
        .unwrap_or_else(|| panic!("no case {id}"))
}

fn reply(text: &str) -> Vec<eval::Recorded> {
    vec![eval::Recorded {
        source: "hand-written".into(),
        reply: text.into(),
    }]
}

/// A graph that prints the string "5" says the right thing without adding anything;
/// `has_any` and `computes` are what refuse it.
#[test]
fn printing_the_answer_as_text_is_not_adding() {
    let mut c = case("build-into-empty-graph");
    c.recorded = reply(
        "Prints the result.\n```patch\nlet s = StringLiteral { value: \"5\" };\nlet out = io.print(value: s.value);\n```",
    );
    let r = eval::replay(
        &c,
        builtin_registry(),
        &flowrs_core::graph::verify::InBrowser,
    );
    assert_eq!(r.outcome, "verified", "{r:?}");
    assert!(!r.passed, "{r:?}");
    assert!(
        r.failures.iter().any(|f| f.contains("no node of any of"))
            && r.failures.iter().any(|f| f.contains("computes 5")),
        "{r:?}"
    );
}

/// `no_edit` accepts the model saying it cannot (`answered`) or the checker refusing
/// every patch (`failed`) — never an edit that went through.
#[test]
fn no_edit_refuses_an_edit() {
    let mut c = case("impossible-fft-fails-honestly");
    c.recorded =
        reply("Prints the message twice.\n```patch\nlet p = PrintLn(input: &n0.value);\n```");
    let r = eval::replay(
        &c,
        builtin_registry(),
        &flowrs_core::graph::verify::InBrowser,
    );
    assert_eq!(r.outcome, "verified", "{r:?}");
    assert!(!r.passed, "{r:?}");
    assert!(
        r.failures
            .iter()
            .any(|f| f.contains("ended verified where the case expects no_edit")),
        "{r:?}"
    );
    let live = eval::replay(
        &case("impossible-fft-fails-honestly-live"),
        builtin_registry(),
        &flowrs_core::graph::verify::InBrowser,
    );
    assert!(live.passed && live.outcome == "answered", "{live:?}");
}