eda1628ec3
A solver-agnostic harness that grades coding-agent task success with an executable test oracle (tests fail before, pass after; pristine test files restored to prevent gaming) and pairs it with measured energy (jouleclaw IOReport, idle-subtracted, sub-window readings flagged unreliable), vendor-billed dollars, and token-estimated cloud energy. Solvers: reference, noop, claude, gemini, and cascade (the real joule_core engine via joule-coded). --passes shows L0 cache amortization. Headline (RESULTS.md): on a realistic operation mix, the cascade and frontier claude both resolve 100%, but the cascade costs ~10x less per task -- mechanical ops close locally at L1 for free, novel ops escalate to the same model once, and repeats are free from the L0 cache. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
30 lines
1.2 KiB
TOML
30 lines
1.2 KiB
TOML
[package]
|
||
name = "joule-bench"
|
||
version = "0.1.0"
|
||
edition = "2021"
|
||
description = "Joule Code coding-agent benchmark: resolution rate × energy/cost 2D frontier over executable coding tasks with a test-execution oracle."
|
||
|
||
[[bin]]
|
||
name = "joule-bench"
|
||
path = "src/main.rs"
|
||
|
||
[features]
|
||
# Measured host energy via jouleclaw (Apple IOReport / Linux RAPL+NVML, no sudo
|
||
# on Apple Silicon) — the same dep and rev agent-joule pins. On by default; build
|
||
# with `--no-default-features` to drop the dep and use the estimator only.
|
||
default = ["measured"]
|
||
measured = ["dep:jouleclaw-energy"]
|
||
|
||
[dependencies]
|
||
clap = { version = "4", features = ["derive"] }
|
||
serde = { version = "1", features = ["derive"] }
|
||
serde_json = "1"
|
||
toml = "0.8"
|
||
anyhow = "1"
|
||
tempfile = "3"
|
||
|
||
[target.'cfg(target_os = "macos")'.dependencies]
|
||
jouleclaw-energy = { git = "https://github.com/Transaction-Science/open-standards.git", rev = "58726a224c72517436bc87ab9f10370cf6b63bb2", features = ["ioreport"], optional = true }
|
||
|
||
[target.'cfg(target_os = "linux")'.dependencies]
|
||
jouleclaw-energy = { git = "https://github.com/Transaction-Science/open-standards.git", rev = "58726a224c72517436bc87ab9f10370cf6b63bb2", features = ["rapl", "nvml"], optional = true }
|