open-standards/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/examples/swebench_live.rs

//! End-to-end: the econ report over instances whose verification is REAL —
//! `git apply` + real test execution via [`GitWorkspaceValidator`], not the
//! offline gold proxy.
//!
//! It builds a throwaway git repo with three planted bugs, defines three
//! SWE-bench-style instances over it (each with its own gold patch + tests),
//! and scores a cascade in joules-and-$ per VERIFIED fix — where "verified"
//! means the candidate patch actually applied and actually flipped the tests.
//! The model is an oracle stand-in that solves two of the three (the third is
//! left unsolved, so it correctly drops out of the denominator).
//!
//! Everything here truly executes (real `git`, real `sh`); the only thing
//! standing in is the patch SOURCE. Swap [`FixedModelTier`] for a real model
//! tier and this is a genuine joules-per-verified-fix run.
//!
//! Run: `cargo run -p jouleclaw-harness-econ --example swebench_live`

use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;

use jouleclaw_cascade::{Cascade, Runtime};
use jouleclaw_harness_econ::swebench::exec::{GitWorkspaceValidator, ShellTestRunner};
use jouleclaw_harness_econ::swebench::{
    gold_model_table, tasks_from, ClassKey, PatchValidator, SweInstance,
};
use jouleclaw_harness_econ::{EconHarness, FixedModelTier};
use jouleclaw_skill::{ConstantSkillInducer, SkillTier};

fn git(dir: &Path, args: &[&str]) -> std::process::Output {
    Command::new("git").current_dir(dir).args(args).output().expect("git runs")
}

/// One planted bug: file `name` holds `bad`, the fix sets it to `good`.
struct Bug {
    name: &'static str,
    bad: &'static str,
    good: &'static str,
}

fn main() {
    if Command::new("git").arg("--version").output().is_err() {
        eprintln!("git unavailable — this live demo needs git + sh");
        return;
    }

    let bugs = [
        Bug { name: "limit.txt", bad: "9", good: "10" },
        Bug { name: "mode.txt", bad: "ro", good: "rw" },
        Bug { name: "color.txt", bad: "red", good: "green" },
    ];

    // ---- build a throwaway repo with all three bugs at the base commit ----
    static C: AtomicU32 = AtomicU32::new(0);
    let dir: PathBuf = std::env::temp_dir()
        .join(format!("jc-swebench-live-{}-{}", std::process::id(), C.fetch_add(1, Ordering::SeqCst)));
    std::fs::create_dir_all(&dir).unwrap();
    git(&dir, &["init", "-q"]);
    git(&dir, &["config", "user.email", "[email protected]"]);
    git(&dir, &["config", "user.name", "t"]);
    for b in &bugs {
        std::fs::write(dir.join(b.name), format!("{}\n", b.bad)).unwrap();
    }
    git(&dir, &["add", "-A"]);
    git(&dir, &["commit", "-q", "-m", "base (with bugs)"]);
    let base = String::from_utf8(git(&dir, &["rev-parse", "HEAD"]).stdout)
        .unwrap()
        .trim()
        .to_string();

    // ---- one SWE-bench-style instance per bug ----
    let mut instances = Vec::new();
    for b in &bugs {
        std::fs::write(dir.join(b.name), format!("{}\n", b.good)).unwrap();
        let gold = String::from_utf8(git(&dir, &["diff", "--", b.name]).stdout).unwrap();
        git(&dir, &["checkout", "--", b.name]);
        instances.push(SweInstance {
            instance_id: format!("local__{}", b.name.trim_end_matches(".txt")),
            repo: "local/app".into(),
            base_commit: base.clone(),
            problem_statement: format!("{} must be {}", b.name, b.good),
            patch: gold,
            test_patch: String::new(),
            fail_to_pass: vec![format!("t::{}_correct", b.name)],
            pass_to_pass: vec![format!("t::{}_exists", b.name)],
            version: None,
            environment_setup_commit: None,
        });
    }

    // ---- real test runner: node id -> shell command, exit 0 = pass ----
    let goods: std::collections::BTreeMap<String, String> = bugs
        .iter()
        .map(|b| (b.name.to_string(), b.good.to_string()))
        .collect();
    let runner = ShellTestRunner::new(move |node: &str| {
        // node ids look like "t::<file>_correct" / "t::<file>_exists"
        let body = node.trim_start_matches("t::");
        if let Some(file) = body.strip_suffix("_correct") {
            let want = goods.get(file).cloned().unwrap_or_default();
            format!("test \"$(cat {file})\" = \"{want}\"")
        } else if let Some(file) = body.strip_suffix("_exists") {
            format!("test -f {file}")
        } else {
            "false".to_string()
        }
    });
    let validator: Arc<dyn PatchValidator> =
        Arc::new(GitWorkspaceValidator::new(&dir, runner));

    let tasks = tasks_from(instances.clone(), validator, ClassKey::PerInstance);

    // Oracle model solves the first two; the third is left unsolved.
    let mut table = gold_model_table(&instances);
    table.pop();

    let harness = EconHarness::new(Box::new(ConstantSkillInducer), 3);
    let report = harness.run(
        move |store| {
            let mut casc = Cascade::new();
            casc.register(Box::new(SkillTier::new(store)));
            casc.register(Box::new(FixedModelTier::new(table.clone(), 0.5)));
            Runtime::new_without_l0(casc)
        },
        &tasks,
        1,
    );

    println!("\n=== SWE-bench-style econ report — REAL git apply + REAL tests ===\n");
    print!("{}", report.to_table());
    println!(
        "\nverified (tests really ran & passed): {}/{}",
        report.verified, report.total
    );
    for row in &report.rows {
        println!(
            "  {:<22} {}  [{}]",
            row.task_id,
            if row.verified { "RESOLVED" } else { "unresolved" },
            row.tier_tag
        );
    }
    println!(
        "joules per verified fix : {:.3e} J   ({:?})",
        report.joules_per_verified_fix, report.energy_provenance
    );
    println!("dollars per verified fix: ${:.6}", report.usd_per_verified_fix);
    println!(
        "\nVerification was operational, not asserted — each RESOLVED row applied a\n\
         patch and the FAIL_TO_PASS / PASS_TO_PASS commands actually executed."
    );

    let _ = std::fs::remove_dir_all(&dir);
}