eda1628ec3
A solver-agnostic harness that grades coding-agent task success with an executable test oracle (tests fail before, pass after; pristine test files restored to prevent gaming) and pairs it with measured energy (jouleclaw IOReport, idle-subtracted, sub-window readings flagged unreliable), vendor-billed dollars, and token-estimated cloud energy. Solvers: reference, noop, claude, gemini, and cascade (the real joule_core engine via joule-coded). --passes shows L0 cache amortization. Headline (RESULTS.md): on a realistic operation mix, the cascade and frontier claude both resolve 100%, but the cascade costs ~10x less per task -- mechanical ops close locally at L1 for free, novel ops escalate to the same model once, and repeats are free from the L0 cache. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
188 lines
No EOL
4.5 KiB
JSON
188 lines
No EOL
4.5 KiB
JSON
{
|
|
"energy_meter": "Measured (ModelBased, jouleclaw host counter, idle≈51.7 W subtracted)",
|
|
"frontier": [
|
|
{
|
|
"cloud_joules_per_resolved": 38445.9,
|
|
"cost_per_resolved": 0.237596125,
|
|
"joules_per_resolved": 0.0,
|
|
"median_wall_ms": 26208,
|
|
"provenance": "ModelBased",
|
|
"reliable_energy_runs": 3,
|
|
"resolution_pct": 66.66666666666667,
|
|
"resolved": 2,
|
|
"solver": "claude",
|
|
"tasks": 3,
|
|
"total_cloud_joules": 76891.8,
|
|
"total_cost_usd": 0.47519225,
|
|
"total_joules": 0.0
|
|
},
|
|
{
|
|
"cloud_joules_per_resolved": null,
|
|
"cost_per_resolved": null,
|
|
"joules_per_resolved": null,
|
|
"median_wall_ms": 0,
|
|
"provenance": "ModelBased",
|
|
"reliable_energy_runs": 0,
|
|
"resolution_pct": 0.0,
|
|
"resolved": 0,
|
|
"solver": "noop",
|
|
"tasks": 3,
|
|
"total_cloud_joules": 0.0,
|
|
"total_cost_usd": -0.0,
|
|
"total_joules": 12.069903
|
|
},
|
|
{
|
|
"cloud_joules_per_resolved": 0.0,
|
|
"cost_per_resolved": null,
|
|
"joules_per_resolved": 4.991112333333334,
|
|
"median_wall_ms": 1,
|
|
"provenance": "ModelBased",
|
|
"reliable_energy_runs": 0,
|
|
"resolution_pct": 100.0,
|
|
"resolved": 3,
|
|
"solver": "reference",
|
|
"tasks": 3,
|
|
"total_cloud_joules": 0.0,
|
|
"total_cost_usd": -0.0,
|
|
"total_joules": 14.973337
|
|
}
|
|
],
|
|
"runs": [
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.2384065,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 38813400000,
|
|
"input_tokens": 183977,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 1009,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"task": "calc",
|
|
"wall_ms": 32111
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.23678575,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 38078400000,
|
|
"input_tokens": 179662,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 1073,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "claude",
|
|
"task": "fizzbuzz",
|
|
"wall_ms": 26208
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": 0.0,
|
|
"energy_reliable": true,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": 0,
|
|
"microjoules": 0,
|
|
"note": "",
|
|
"output_tokens": 0,
|
|
"provenance": "ModelBased",
|
|
"resolved": false,
|
|
"solver": "claude",
|
|
"task": "slicer",
|
|
"wall_ms": 1959
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": null,
|
|
"energy_reliable": false,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 5584226,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "reference",
|
|
"task": "calc",
|
|
"wall_ms": 0
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": null,
|
|
"energy_reliable": false,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 5642953,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "reference",
|
|
"task": "fizzbuzz",
|
|
"wall_ms": 1
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": null,
|
|
"energy_reliable": false,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 3746158,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"provenance": "ModelBased",
|
|
"resolved": true,
|
|
"solver": "reference",
|
|
"task": "slicer",
|
|
"wall_ms": 1
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": null,
|
|
"energy_reliable": false,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 4521968,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"provenance": "ModelBased",
|
|
"resolved": false,
|
|
"solver": "noop",
|
|
"task": "calc",
|
|
"wall_ms": 0
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": null,
|
|
"energy_reliable": false,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 3458960,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"provenance": "ModelBased",
|
|
"resolved": false,
|
|
"solver": "noop",
|
|
"task": "fizzbuzz",
|
|
"wall_ms": 0
|
|
},
|
|
{
|
|
"baseline_failed": true,
|
|
"cost_usd": null,
|
|
"energy_reliable": false,
|
|
"est_cloud_uj": 0,
|
|
"input_tokens": null,
|
|
"microjoules": 4088975,
|
|
"note": "",
|
|
"output_tokens": null,
|
|
"provenance": "ModelBased",
|
|
"resolved": false,
|
|
"solver": "noop",
|
|
"task": "slicer",
|
|
"wall_ms": 0
|
|
}
|
|
],
|
|
"suite": "tasks/smoke"
|
|
} |