eda1628ec3
A solver-agnostic harness that grades coding-agent task success with an executable test oracle (tests fail before, pass after; pristine test files restored to prevent gaming) and pairs it with measured energy (jouleclaw IOReport, idle-subtracted, sub-window readings flagged unreliable), vendor-billed dollars, and token-estimated cloud energy. Solvers: reference, noop, claude, gemini, and cascade (the real joule_core engine via joule-coded). --passes shows L0 cache amortization. Headline (RESULTS.md): on a realistic operation mix, the cascade and frontier claude both resolve 100%, but the cascade costs ~10x less per task -- mechanical ops close locally at L1 for free, novel ops escalate to the same model once, and repeats are free from the L0 cache. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
378 lines
No EOL
8.9 KiB
JSON
378 lines
No EOL
8.9 KiB
JSON
{
|
|
"energy_meter": "Measured (ModelBased, jouleclaw host counter, idle≈134.9 W subtracted)",
|
|
"frontier": [
|
|
{
|
|
"cloud_joules_per_resolved": 0.0,
|
|
"cost_per_resolved": 0.020660274999999995,
|
|
"joules_per_resolved": 0.0,
|
|
"median_wall_ms": 1298,
|
|
"provenance": "ModelBased",
|
|
"reliable_energy_runs": 10,
|
|
"resolution_pct": 100.0,
|
|
"resolved": 10,
|
|
"solver": "cascade",
|
|
"tasks": 10,
|
|
"total_cloud_joules": 0.0,
|
|
"total_cost_usd": 0.20660274999999995,
|
|
"total_joules": 0.0
|
|
},
|
|
{
|
|
"cloud_joules_per_resolved": 29302.379999999997,
|
|
"cost_per_resolved": 0.20433379999999995,
|
|
"joules_per_resolved": 0.0,
|
|
"median_wall_ms": 22131,
|
|
"provenance": "ModelBased",
|
|
"reliable_energy_runs": 10,
|
|
"resolution_pct": 100.0,
|
|
"resolved": 10,
|
|
"solver": "claude",
|
|
"tasks": 10,
|
|
"total_cloud_joules": 293023.8,
|
|
"total_cost_usd": 2.0433379999999994,
|
|
"total_joules": 0.0
|
|
}
|
|
],
|
|
"runs": [
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L1",
|
|
"task": "const",
|
|
"wall_ms": 1313
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L1",
|
|
"task": "greeting",
|
|
"wall_ms": 1298
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.20660274999999995,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L4",
|
|
"task": "logic",
|
|
"wall_ms": 23145
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L1",
|
|
"task": "rename",
|
|
"wall_ms": 1778
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L1",
|
|
"task": "typo",
|
|
"wall_ms": 1778
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.188752,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 24369800000,
|
|
"input_tokens": 116399,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 545,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "const",
|
|
"wall_ms": 16374
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.1864925,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 24188800000,
|
|
"input_tokens": 115974,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 497,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "greeting",
|
|
"wall_ms": 16516
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.213112,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 31848200000,
|
|
"input_tokens": 152301,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 694,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "logic",
|
|
"wall_ms": 222593
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.215365,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 32044000000,
|
|
"input_tokens": 152610,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 761,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "rename",
|
|
"wall_ms": 20984
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.2314895,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 38248800000,
|
|
"input_tokens": 182804,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 844,
|
|
"pass": 1,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "typo",
|
|
"wall_ms": 23024
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L0",
|
|
"task": "const",
|
|
"wall_ms": 167
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L0",
|
|
"task": "greeting",
|
|
"wall_ms": 166
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L0",
|
|
"task": "logic",
|
|
"wall_ms": 165
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L0",
|
|
"task": "rename",
|
|
"wall_ms": 168
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "cascade",
|
|
"solver_note": "L0",
|
|
"task": "typo",
|
|
"wall_ms": 170
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.18969975,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 24445000000,
|
|
"input_tokens": 116495,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 573,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "const",
|
|
"wall_ms": 23325
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.185425,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 24103000000,
|
|
"input_tokens": 115845,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 467,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "greeting",
|
|
"wall_ms": 20524
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.2105145,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 31648200000,
|
|
"input_tokens": 152031,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 621,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "logic",
|
|
"wall_ms": 222810
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.21362975,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 31256600000,
|
|
"input_tokens": 148463,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 782,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "rename",
|
|
"wall_ms": 19798
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.208858,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 30871400000,
|
|
"input_tokens": 147157,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 720,
|
|
"pass": 2,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"solver_note": null,
|
|
"task": "typo",
|
|
"wall_ms": 22131
|
|
}
|
|
],
|
|
"suite": "tasks/mix"
|
|
} |