open-standards/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/examples/swebench_econ.rs

//! Run the economics report against the **SWE-bench / SWE-bench-Pro** instance
//! format: joules-and-dollars per _verified_ fix on the standardized set.
//!
//! ```text
//! cargo run -p jouleclaw-harness-econ --example swebench_econ            # embedded sample
//! cargo run -p jouleclaw-harness-econ --example swebench_econ -- FILE.jsonl  # a real dump
//! ```
//!
//! With no argument it uses a tiny embedded sample in the real schema. Point it
//! at a SWE-bench-Pro JSONL dump and it loads that instead. The model here is an
//! **oracle stand-in** (emits the gold patch) so the run is self-contained and
//! shows the harness mechanics; swap in a real model tier + a container-backed
//! [`PatchValidator`] for a genuine score. Verification uses the offline
//! [`GoldPatchOracle`] proxy — see the module docs for the honest caveat.

use std::sync::Arc;

use jouleclaw_cascade::{Cascade, Runtime};
use jouleclaw_harness_econ::swebench::{
    gold_model_table, load_jsonl, parse_jsonl, tasks_from, ClassKey, GoldPatchOracle,
    PatchValidator, SweInstance,
};
use jouleclaw_harness_econ::{EconHarness, FixedModelTier};
use jouleclaw_skill::{ConstantSkillInducer, SkillTier};

/// A small embedded sample in the standardized schema (synthetic — clearly not
/// real SWE-bench data; just enough to exercise the loader + report).
const SAMPLE: &str = r#"
{"instance_id":"acme__widget-101","repo":"acme/widget","base_commit":"abc123","problem_statement":"range() loop drops the last element (off-by-one)","patch":"--- a/w.py\n+++ b/w.py\n@@\n-    for i in range(n-1):\n+    for i in range(n):","test_patch":"--- a/t.py","FAIL_TO_PASS":"[\"t.py::test_full_range\"]","PASS_TO_PASS":"[\"t.py::test_empty\"]","version":"1.2"}
{"instance_id":"acme__widget-102","repo":"acme/widget","base_commit":"def456","problem_statement":"first() panics on empty input","patch":"--- a/w.py\n+++ b/w.py\n@@\n-    return xs[0]\n+    return xs[0] if xs else None","test_patch":"--- a/t.py","FAIL_TO_PASS":["t.py::test_empty_input"],"PASS_TO_PASS":[]}
{"instance_id":"acme__gadget-7","repo":"acme/gadget","base_commit":"99aa00","problem_statement":"timezone offset applied twice when parsing ISO dates","patch":"--- a/d.py\n+++ b/d.py\n@@\n-    return t + tz + tz\n+    return t + tz","test_patch":"--- a/t.py","FAIL_TO_PASS":["t.py::test_iso_tz"],"PASS_TO_PASS":["t.py::test_naive"]}
"#;

fn main() {
    let path = std::env::args().nth(1);
    let instances: Vec<SweInstance> = match &path {
        Some(p) => match load_jsonl(p) {
            Ok(v) => {
                println!("loaded {} instances from {p}", v.len());
                v
            }
            Err(e) => {
                eprintln!("failed to load {p}: {e}");
                std::process::exit(1);
            }
        },
        None => {
            println!("(no path given — using embedded sample; pass a SWE-bench-Pro JSONL to score a real set)");
            parse_jsonl(SAMPLE).expect("embedded sample parses")
        }
    };

    let n = instances.len();
    let validator: Arc<dyn PatchValidator> = Arc::new(GoldPatchOracle);
    let tasks = tasks_from(instances.clone(), validator, ClassKey::PerInstance);

    // Oracle model that "solves" all but the LAST instance — so one task is
    // unsolved and correctly drops out of the verified denominator.
    let mut table = gold_model_table(&instances);
    if !table.is_empty() {
        table.pop();
    }

    let harness = EconHarness::new(Box::new(ConstantSkillInducer), 3);
    let report = harness.run(
        move |store| {
            let mut casc = Cascade::new();
            casc.register(Box::new(SkillTier::new(store)));
            // 0.5 J/dispatch stands in for a frontier patch generation.
            casc.register(Box::new(FixedModelTier::new(table.clone(), 0.5)));
            Runtime::new_without_l0(casc)
        },
        &tasks,
        1, // SWE-bench is one-shot
    );

    println!("\n=== SWE-bench-Pro econ report — {n} instances (one-shot) ===\n");
    print!("{}", report.to_table());
    println!(
        "\nresolved (verified): {}/{}   [verification: GoldPatchOracle offline proxy]",
        report.verified, report.total
    );
    println!(
        "joules per verified fix : {:.3e} J   ({:?})",
        report.joules_per_verified_fix, report.energy_provenance
    );
    println!("dollars per verified fix: ${:.6}", report.usd_per_verified_fix);
    println!(
        "\nThe denominator is fixes that PASS verification, not attempts — so this is\n\
         cost-per-fix-that-holds-up. Swap in a container-backed PatchValidator and a\n\
         real model tier for a genuine SWE-bench-Pro score with a joules axis."
    );
}