//! End-to-end: the econ report over instances whose verification is REAL — //! `git apply` + real test execution via [`GitWorkspaceValidator`], not the //! offline gold proxy. //! //! It builds a throwaway git repo with three planted bugs, defines three //! SWE-bench-style instances over it (each with its own gold patch + tests), //! and scores a cascade in joules-and-$ per VERIFIED fix — where "verified" //! means the candidate patch actually applied and actually flipped the tests. //! The model is an oracle stand-in that solves two of the three (the third is //! left unsolved, so it correctly drops out of the denominator). //! //! Everything here truly executes (real `git`, real `sh`); the only thing //! standing in is the patch SOURCE. Swap [`FixedModelTier`] for a real model //! tier and this is a genuine joules-per-verified-fix run. //! //! Run: `cargo run -p jouleclaw-harness-econ --example swebench_live` use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use jouleclaw_cascade::{Cascade, Runtime}; use jouleclaw_harness_econ::swebench::exec::{GitWorkspaceValidator, ShellTestRunner}; use jouleclaw_harness_econ::swebench::{ gold_model_table, tasks_from, ClassKey, PatchValidator, SweInstance, }; use jouleclaw_harness_econ::{EconHarness, FixedModelTier}; use jouleclaw_skill::{ConstantSkillInducer, SkillTier}; fn git(dir: &Path, args: &[&str]) -> std::process::Output { Command::new("git").current_dir(dir).args(args).output().expect("git runs") } /// One planted bug: file `name` holds `bad`, the fix sets it to `good`. struct Bug { name: &'static str, bad: &'static str, good: &'static str, } fn main() { if Command::new("git").arg("--version").output().is_err() { eprintln!("git unavailable — this live demo needs git + sh"); return; } let bugs = [ Bug { name: "limit.txt", bad: "9", good: "10" }, Bug { name: "mode.txt", bad: "ro", good: "rw" }, Bug { name: "color.txt", bad: "red", good: "green" }, ]; // ---- build a throwaway repo with all three bugs at the base commit ---- static C: AtomicU32 = AtomicU32::new(0); let dir: PathBuf = std::env::temp_dir() .join(format!("jc-swebench-live-{}-{}", std::process::id(), C.fetch_add(1, Ordering::SeqCst))); std::fs::create_dir_all(&dir).unwrap(); git(&dir, &["init", "-q"]); git(&dir, &["config", "user.email", "t@t.test"]); git(&dir, &["config", "user.name", "t"]); for b in &bugs { std::fs::write(dir.join(b.name), format!("{}\n", b.bad)).unwrap(); } git(&dir, &["add", "-A"]); git(&dir, &["commit", "-q", "-m", "base (with bugs)"]); let base = String::from_utf8(git(&dir, &["rev-parse", "HEAD"]).stdout) .unwrap() .trim() .to_string(); // ---- one SWE-bench-style instance per bug ---- let mut instances = Vec::new(); for b in &bugs { std::fs::write(dir.join(b.name), format!("{}\n", b.good)).unwrap(); let gold = String::from_utf8(git(&dir, &["diff", "--", b.name]).stdout).unwrap(); git(&dir, &["checkout", "--", b.name]); instances.push(SweInstance { instance_id: format!("local__{}", b.name.trim_end_matches(".txt")), repo: "local/app".into(), base_commit: base.clone(), problem_statement: format!("{} must be {}", b.name, b.good), patch: gold, test_patch: String::new(), fail_to_pass: vec![format!("t::{}_correct", b.name)], pass_to_pass: vec![format!("t::{}_exists", b.name)], version: None, environment_setup_commit: None, }); } // ---- real test runner: node id -> shell command, exit 0 = pass ---- let goods: std::collections::BTreeMap = bugs .iter() .map(|b| (b.name.to_string(), b.good.to_string())) .collect(); let runner = ShellTestRunner::new(move |node: &str| { // node ids look like "t::_correct" / "t::_exists" let body = node.trim_start_matches("t::"); if let Some(file) = body.strip_suffix("_correct") { let want = goods.get(file).cloned().unwrap_or_default(); format!("test \"$(cat {file})\" = \"{want}\"") } else if let Some(file) = body.strip_suffix("_exists") { format!("test -f {file}") } else { "false".to_string() } }); let validator: Arc = Arc::new(GitWorkspaceValidator::new(&dir, runner)); let tasks = tasks_from(instances.clone(), validator, ClassKey::PerInstance); // Oracle model solves the first two; the third is left unsolved. let mut table = gold_model_table(&instances); table.pop(); let harness = EconHarness::new(Box::new(ConstantSkillInducer), 3); let report = harness.run( move |store| { let mut casc = Cascade::new(); casc.register(Box::new(SkillTier::new(store))); casc.register(Box::new(FixedModelTier::new(table.clone(), 0.5))); Runtime::new_without_l0(casc) }, &tasks, 1, ); println!("\n=== SWE-bench-style econ report — REAL git apply + REAL tests ===\n"); print!("{}", report.to_table()); println!( "\nverified (tests really ran & passed): {}/{}", report.verified, report.total ); for row in &report.rows { println!( " {:<22} {} [{}]", row.task_id, if row.verified { "RESOLVED" } else { "unresolved" }, row.tier_tag ); } println!( "joules per verified fix : {:.3e} J ({:?})", report.joules_per_verified_fix, report.energy_provenance ); println!("dollars per verified fix: ${:.6}", report.usd_per_verified_fix); println!( "\nVerification was operational, not asserted — each RESOLVED row applied a\n\ patch and the FAIL_TO_PASS / PASS_TO_PASS commands actually executed." ); let _ = std::fs::remove_dir_all(&dir); }