//! Run the economics report against the **SWE-bench / SWE-bench-Pro** instance //! format: joules-and-dollars per _verified_ fix on the standardized set. //! //! ```text //! cargo run -p jouleclaw-harness-econ --example swebench_econ # embedded sample //! cargo run -p jouleclaw-harness-econ --example swebench_econ -- FILE.jsonl # a real dump //! ``` //! //! With no argument it uses a tiny embedded sample in the real schema. Point it //! at a SWE-bench-Pro JSONL dump and it loads that instead. The model here is an //! **oracle stand-in** (emits the gold patch) so the run is self-contained and //! shows the harness mechanics; swap in a real model tier + a container-backed //! [`PatchValidator`] for a genuine score. Verification uses the offline //! [`GoldPatchOracle`] proxy — see the module docs for the honest caveat. use std::sync::Arc; use jouleclaw_cascade::{Cascade, Runtime}; use jouleclaw_harness_econ::swebench::{ gold_model_table, load_jsonl, parse_jsonl, tasks_from, ClassKey, GoldPatchOracle, PatchValidator, SweInstance, }; use jouleclaw_harness_econ::{EconHarness, FixedModelTier}; use jouleclaw_skill::{ConstantSkillInducer, SkillTier}; /// A small embedded sample in the standardized schema (synthetic — clearly not /// real SWE-bench data; just enough to exercise the loader + report). const SAMPLE: &str = r#" {"instance_id":"acme__widget-101","repo":"acme/widget","base_commit":"abc123","problem_statement":"range() loop drops the last element (off-by-one)","patch":"--- a/w.py\n+++ b/w.py\n@@\n- for i in range(n-1):\n+ for i in range(n):","test_patch":"--- a/t.py","FAIL_TO_PASS":"[\"t.py::test_full_range\"]","PASS_TO_PASS":"[\"t.py::test_empty\"]","version":"1.2"} {"instance_id":"acme__widget-102","repo":"acme/widget","base_commit":"def456","problem_statement":"first() panics on empty input","patch":"--- a/w.py\n+++ b/w.py\n@@\n- return xs[0]\n+ return xs[0] if xs else None","test_patch":"--- a/t.py","FAIL_TO_PASS":["t.py::test_empty_input"],"PASS_TO_PASS":[]} {"instance_id":"acme__gadget-7","repo":"acme/gadget","base_commit":"99aa00","problem_statement":"timezone offset applied twice when parsing ISO dates","patch":"--- a/d.py\n+++ b/d.py\n@@\n- return t + tz + tz\n+ return t + tz","test_patch":"--- a/t.py","FAIL_TO_PASS":["t.py::test_iso_tz"],"PASS_TO_PASS":["t.py::test_naive"]} "#; fn main() { let path = std::env::args().nth(1); let instances: Vec = match &path { Some(p) => match load_jsonl(p) { Ok(v) => { println!("loaded {} instances from {p}", v.len()); v } Err(e) => { eprintln!("failed to load {p}: {e}"); std::process::exit(1); } }, None => { println!("(no path given — using embedded sample; pass a SWE-bench-Pro JSONL to score a real set)"); parse_jsonl(SAMPLE).expect("embedded sample parses") } }; let n = instances.len(); let validator: Arc = Arc::new(GoldPatchOracle); let tasks = tasks_from(instances.clone(), validator, ClassKey::PerInstance); // Oracle model that "solves" all but the LAST instance — so one task is // unsolved and correctly drops out of the verified denominator. let mut table = gold_model_table(&instances); if !table.is_empty() { table.pop(); } let harness = EconHarness::new(Box::new(ConstantSkillInducer), 3); let report = harness.run( move |store| { let mut casc = Cascade::new(); casc.register(Box::new(SkillTier::new(store))); // 0.5 J/dispatch stands in for a frontier patch generation. casc.register(Box::new(FixedModelTier::new(table.clone(), 0.5))); Runtime::new_without_l0(casc) }, &tasks, 1, // SWE-bench is one-shot ); println!("\n=== SWE-bench-Pro econ report — {n} instances (one-shot) ===\n"); print!("{}", report.to_table()); println!( "\nresolved (verified): {}/{} [verification: GoldPatchOracle offline proxy]", report.verified, report.total ); println!( "joules per verified fix : {:.3e} J ({:?})", report.joules_per_verified_fix, report.energy_provenance ); println!("dollars per verified fix: ${:.6}", report.usd_per_verified_fix); println!( "\nThe denominator is fixes that PASS verification, not attempts — so this is\n\ cost-per-fix-that-holds-up. Swap in a container-backed PatchValidator and a\n\ real model tier for a genuine SWE-bench-Pro score with a joules axis." ); }