9258a0739f
The apps branch's documents join the version contract. The answer layer's documents (wai.quantum.answer/1: an answer read, carried out, a seal's request and result, a stored answer checked), described as a draft whose form may change before the extension settles; a correctness record and its corpus (wai.quantum.telemetry/1); the views page's manifest (wai.quantum.views-bundle/1); and the command line's harness report, corpus and token counts (wai.quantum.eval-report/1, eval-answers/1, token-cost/1). The engine's test holds every answer of the layer's conformance set (read, carried out, half arrived, and stored and verified), records, the corpus and the committed manifest to them; the eval tests hold a scripted run's report, a model server's, the corpus and the committed token counts; check_schemas.py holds the committed documents too. The tool and diagnostic schemas take the answer layer's outcomes, dropped and defaulted. The result view's own tool, poll_quantum_job, listed only to a host that draws views, answers to the job's schema, and the strict test holds it. The basis rename reaches the views: a compile's words and its view name the gate set by its basis (both read the certificate's old target), and a test in each holds it. The service's job summary now lives in engine::views (the apps branch moved it there); it carries the durable branch's set summary and the processor packages' energy wording with it. The answer set's stored seal was made at 0.6.16; the values it seals name their engine, so it is written again at 0.7.0 (WAI_ANSWER_BUNDLE_WRITE=1). The views crate's lock moves to wai-quantum 0.7.0. Co-Authored-By: Claude Opus 5.5 <[email protected]>
763 lines
40 KiB
Rust
763 lines
40 KiB
Rust
//! `wai-quantum eval`: a model asked the tasks of a set and its answers
|
|
//! scored by the engine, as a report on this machine.
|
|
//!
|
|
//! ```text
|
|
//! wai-quantum eval run --tasks tasks.json --model http://127.0.0.1:8080/v1 --out report.json
|
|
//! wai-quantum eval run --tasks tasks.json --scripted answers.json # the harness itself, no model
|
|
//! wai-quantum eval run --tasks tasks.json --model URL --format views # answers in the answer view layer
|
|
//! wai-quantum eval corpus --tasks tasks.json --out answers.json # each answer written four ways
|
|
//! ```
|
|
//!
|
|
//! A task (`wai/quantum-conformance/eval/tasks.json`, in the conformance
|
|
//! format) says what to ask and the intent the answer is held to. For each,
|
|
//! the model is asked once, at temperature 0, and its answer scored:
|
|
//!
|
|
//! - **valid**: the engine reads its program (and, in the answer view layer,
|
|
//! the answer has no error), else the codes that say why;
|
|
//! - **the verdict**: the program run with the task's intent, and the level
|
|
//! the engine checked it to (proof, evidence, executed, parsed, refused)
|
|
//! with whether the intent holds; a task is **solved** when it holds by
|
|
//! proof or evidence;
|
|
//! - **joules**: with `--energy-pid`, the model server's energy over the
|
|
//! call, where this host meters it (on macOS, the CPU energy the kernel
|
|
//! counts for that process; its GPU is not counted), and otherwise why not.
|
|
//!
|
|
//! The report is one model on one task set. It ranks nothing, and is kept
|
|
//! where it is written. The model server is one on this machine or a
|
|
//! private network; another is refused unless `--allow-remote` says so (a
|
|
//! service may charge for each call). No key or token is read or sent.
|
|
|
|
use serde_json::{Value, json};
|
|
use std::path::{Path, PathBuf};
|
|
use wai_quantum::engine::{answer, guide, job, verdict};
|
|
|
|
#[derive(clap::Subcommand)]
|
|
pub enum Cmd {
|
|
/// Ask a model each task of a set and score its answers with the engine: whether its program reads, the verdict's level and whether the intent holds, and the joules per solved task where the model server's energy is metered
|
|
Run(RunArgs),
|
|
/// Write each task's answer four ways (the program alone, the answer view layer whole and with the request alone, a job's JSON) for measuring what each costs a model in tokens
|
|
Corpus(CorpusArgs),
|
|
}
|
|
|
|
/// How a model answers.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)]
|
|
pub enum Format {
|
|
/// The program, in a fenced qasm block
|
|
Qasm,
|
|
/// The answer view layer: the program and the views that show it
|
|
Views,
|
|
}
|
|
|
|
impl Format {
|
|
fn name(self) -> &'static str {
|
|
match self {
|
|
Format::Qasm => "qasm",
|
|
Format::Views => "views",
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(clap::Args)]
|
|
pub struct RunArgs {
|
|
/// The task set, in the conformance format (wai/quantum-conformance/eval/tasks.json)
|
|
#[arg(long, value_name = "FILE")]
|
|
pub tasks: PathBuf,
|
|
/// The model server: a chat-completions endpoint on this machine or a private network, as http://127.0.0.1:8080/v1
|
|
#[arg(long, value_name = "URL", conflicts_with = "scripted", required_unless_present = "scripted")]
|
|
pub model: Option<String>,
|
|
/// The model's name as its server knows it (default: the server's own)
|
|
#[arg(long, value_name = "NAME")]
|
|
pub name: Option<String>,
|
|
/// Answers written beforehand, as {"task": "answer", …}: the harness itself tested, with no model asked
|
|
#[arg(long, value_name = "FILE")]
|
|
pub scripted: Option<PathBuf>,
|
|
/// How the model is asked to answer: qasm (the program) or views (the answer view layer)
|
|
#[arg(long, value_enum, default_value_t = Format::Qasm)]
|
|
pub format: Format,
|
|
/// The model server's process, whose energy over each call is metered where this host can (macOS)
|
|
#[arg(long, value_name = "PID")]
|
|
pub energy_pid: Option<u32>,
|
|
/// Allow a model server that is neither on this machine nor on a private network (it may charge for each call)
|
|
#[arg(long)]
|
|
pub allow_remote: bool,
|
|
/// The most tokens the model may answer with, per task
|
|
#[arg(long, default_value_t = 4096)]
|
|
pub max_tokens: u32,
|
|
/// After an answer the engine cannot read, send its errors (the codes and lines, never the notes) back to the model this many times for a corrected answer
|
|
#[arg(long, default_value_t = 0)]
|
|
pub repair_rounds: u32,
|
|
/// Only these tasks (repeatable)
|
|
#[arg(long = "task", value_name = "NAME")]
|
|
pub only: Vec<String>,
|
|
/// Write the report here (default: standard output)
|
|
#[arg(long, short, value_name = "FILE")]
|
|
pub out: Option<PathBuf>,
|
|
}
|
|
|
|
#[derive(clap::Args)]
|
|
pub struct CorpusArgs {
|
|
/// The task set
|
|
#[arg(long, value_name = "FILE")]
|
|
pub tasks: PathBuf,
|
|
/// Write the corpus here (default: standard output)
|
|
#[arg(long, short, value_name = "FILE")]
|
|
pub out: Option<PathBuf>,
|
|
}
|
|
|
|
/// The report's format.
|
|
pub const REPORT: &str = "wai.quantum.eval-report/1";
|
|
/// The corpus's format.
|
|
pub const ANSWERS: &str = "wai.quantum.eval-answers/1";
|
|
|
|
pub fn main(cmd: &Cmd) -> Result<(), String> {
|
|
match cmd {
|
|
Cmd::Run(a) => {
|
|
let report = run(a)?;
|
|
let text = serde_json::to_string_pretty(&report).map_err(|e| e.to_string())? + "\n";
|
|
write_out(a.out.as_deref(), &text)?;
|
|
let s = &report["summary"];
|
|
eprintln!("{} tasks: {} valid, {} solved; {}", s["tasks"], s["valid"], s["solved"], s["joules"]["per_solved"].as_f64().map(|j| format!("{j:.3} J per solved task ({})", s["joules"]["covers"].as_str().unwrap_or(""))).unwrap_or_else(|| format!("energy unmetered: {}", s["joules"]["unmetered"].as_str().unwrap_or(""))));
|
|
Ok(())
|
|
}
|
|
Cmd::Corpus(a) => {
|
|
let c = corpus(&load_tasks(&a.tasks)?)?;
|
|
write_out(a.out.as_deref(), &(serde_json::to_string_pretty(&c).map_err(|e| e.to_string())? + "\n"))
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write_out(path: Option<&Path>, text: &str) -> Result<(), String> {
|
|
match path {
|
|
Some(p) => std::fs::write(p, text).map_err(|e| format!("{}: {e}", p.display())),
|
|
None => {
|
|
print!("{text}");
|
|
Ok(())
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A task set's cases.
|
|
pub fn load_tasks(path: &Path) -> Result<Vec<Value>, String> {
|
|
let v: Value = serde_json::from_str(&std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?).map_err(|e| format!("{}: not JSON: {e}", path.display()))?;
|
|
let cases = v["cases"].as_array().ok_or(format!("{}: no cases", path.display()))?;
|
|
for c in cases {
|
|
if !c["name"].is_string() || !c["ask"].is_string() || c.get("intent").is_none() {
|
|
return Err(format!("{}: a task takes name, ask and intent: {c}", path.display()));
|
|
}
|
|
}
|
|
Ok(cases.clone())
|
|
}
|
|
|
|
/// What a model is told, by the format it answers in.
|
|
pub fn system_prompt(format: Format) -> String {
|
|
match format {
|
|
Format::Qasm => format!(
|
|
"You write quantum programs in OpenQASM 3. {} Answer with the whole program in one ```qasm block and nothing else: no explanation, and no counts, probabilities or other figures, which the engine computes.",
|
|
guide::SUMMARY
|
|
),
|
|
Format::Views => format!(
|
|
"{}\nAnswer with the program the task asks for, as a ```qasm block named after its statement, and views that show it; include a Verdict(@Check(...)) of what the task asks. Write nothing outside the layer.",
|
|
answer::prompt()
|
|
),
|
|
}
|
|
}
|
|
|
|
/// Whether a model server's address is on this machine or a private network.
|
|
pub fn private(url: &str) -> bool {
|
|
let rest = url.split_once("://").map_or(url, |(_, r)| r);
|
|
let host_port = rest.split('/').next().unwrap_or("");
|
|
let host = if let Some(h) = host_port.strip_prefix('[') { h.split(']').next().unwrap_or("") } else { host_port.rsplit_once(':').map_or(host_port, |(h, _)| h) };
|
|
if host == "localhost" || host == "::1" {
|
|
return true;
|
|
}
|
|
match host.parse::<std::net::Ipv4Addr>() {
|
|
Ok(ip) => {
|
|
let [a, b, ..] = ip.octets();
|
|
ip.is_loopback() || ip.is_private() || (a == 100 && (64..128).contains(&b))
|
|
}
|
|
Err(_) => false,
|
|
}
|
|
}
|
|
|
|
/// The program in an answer, and what the answer layer says of it.
|
|
pub struct Extracted {
|
|
pub program: Option<String>,
|
|
/// The answer layer's codes (views only).
|
|
pub codes: Vec<String>,
|
|
/// The answer layer's errors (views only).
|
|
pub errors: usize,
|
|
}
|
|
|
|
/// The answer a model gave, read for its program.
|
|
pub fn extract(text: &str, format: Format) -> Extracted {
|
|
// A model that thinks aloud says so between tags; that is not its answer.
|
|
let mut t = text.to_string();
|
|
while let (Some(a), Some(b)) = (t.find("<think>"), t.find("</think>")) {
|
|
if b < a {
|
|
break;
|
|
}
|
|
t.replace_range(a..b + "</think>".len(), "");
|
|
}
|
|
match format {
|
|
Format::Views => {
|
|
let a = answer::parse(t.trim_start());
|
|
let program = a.stmts.iter().find(|c| c.kind == answer::Kind::Program).and_then(|c| match &c.stmt.body {
|
|
answer::Body::Program { qasm, .. } => Some(qasm.clone()),
|
|
_ => None,
|
|
});
|
|
Extracted { program, codes: a.diags.iter().map(|d| d.code.to_string()).collect(), errors: a.errors().len() }
|
|
}
|
|
Format::Qasm => {
|
|
let mut program = None;
|
|
let mut rest = t.as_str();
|
|
while let Some(i) = rest.find("```") {
|
|
let after = &rest[i + 3..];
|
|
let Some(nl) = after.find('\n') else { break };
|
|
let lang = after[..nl].trim().to_ascii_lowercase();
|
|
let body = &after[nl + 1..];
|
|
let Some(end) = body.find("```") else { break };
|
|
let code = &body[..end];
|
|
if (lang.is_empty() || lang.starts_with("qasm") || lang.starts_with("openqasm")) && (code.contains("OPENQASM") || code.contains("qubit")) {
|
|
program = Some(code.to_string());
|
|
break;
|
|
}
|
|
rest = &body[end + 3..];
|
|
}
|
|
if program.is_none() && t.trim_start().starts_with("OPENQASM") {
|
|
program = Some(t.trim().to_string() + "\n");
|
|
}
|
|
Extracted { program, codes: Vec::new(), errors: 0 }
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One answer scored against its task: valid, the codes, the verdict's level
|
|
/// and whether the intent holds, and whether the task is solved.
|
|
pub fn score(task: &Value, text: &str, format: Format) -> Value {
|
|
let x = extract(text, format);
|
|
let mut out = json!({ "valid": false, "codes": x.codes, "level": null, "holds": null, "solved": false, "program": x.program });
|
|
let Some(program) = x.program else {
|
|
out["why"] = json!("the answer holds no program");
|
|
return out;
|
|
};
|
|
let lint = job::lint(&program);
|
|
if lint["ok"] != true {
|
|
if let Some(c) = lint["code"].as_str()
|
|
&& let Some(list) = out["codes"].as_array_mut()
|
|
{
|
|
list.push(json!(c));
|
|
}
|
|
out["why"] = lint["error"].clone();
|
|
out["level"] = json!("refused");
|
|
return out;
|
|
}
|
|
out["valid"] = json!(x.errors == 0);
|
|
if x.errors > 0 {
|
|
out["why"] = json!(format!("the answer has {} error{} in the answer view layer", x.errors, if x.errors == 1 { "" } else { "s" }));
|
|
}
|
|
let mut req = json!({ "qasm": program, "verdict": true, "intent": task["intent"] });
|
|
for (k, v) in task["run"].as_object().into_iter().flatten() {
|
|
req[k] = v.clone();
|
|
}
|
|
if req.get("shots").is_none() {
|
|
req["shots"] = json!(100);
|
|
}
|
|
let v = match job::plan_and_run(&req, 1 << 30) {
|
|
Ok((_, result)) => result["verdict"].clone(),
|
|
Err(e) => verdict::of_refusal(&req, &e),
|
|
};
|
|
out["level"] = v["level"].clone();
|
|
out["holds"] = v["holds"].clone();
|
|
out["statement"] = v["statement"].clone();
|
|
if let Some(r) = v["refusal"]["codes"].as_array()
|
|
&& let Some(list) = out["codes"].as_array_mut()
|
|
{
|
|
list.extend(r.iter().cloned());
|
|
}
|
|
out["solved"] = json!(v["holds"] == true && matches!(v["level"].as_str(), Some("proof" | "evidence")) && out["valid"] == true);
|
|
out
|
|
}
|
|
|
|
/// What the engine could not read in an answer, as the lines a model is
|
|
/// sent back to repair it: the answer layer's errors, and the reader's
|
|
/// refusal of its program; nothing for an answer that reads.
|
|
pub fn errors(text: &str, format: Format) -> Vec<String> {
|
|
let x = extract(text, format);
|
|
let mut out = Vec::new();
|
|
if format == Format::Views {
|
|
out.extend(answer::parse(text).errors());
|
|
}
|
|
match &x.program {
|
|
None => out.push(match format {
|
|
Format::Qasm => "the answer holds no program: give the whole program in one ```qasm block".into(),
|
|
Format::Views => "the answer holds no program: give it in a fenced block that names it, ```qasm NAME".into(),
|
|
}),
|
|
Some(p) => {
|
|
let lint = job::lint(p);
|
|
if lint["ok"] != true {
|
|
out.push(lint["error"].as_str().unwrap_or("the program does not read").to_string());
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// A model's answer to one task: its text, the tokens the server counted,
|
|
/// and the seconds it took.
|
|
struct Asked {
|
|
text: String,
|
|
usage: Value,
|
|
seconds: f64,
|
|
}
|
|
|
|
#[cfg(feature = "service")]
|
|
fn ask(endpoint: &str, name: Option<&str>, messages: &[Value], max_tokens: u32) -> Result<Asked, String> {
|
|
use std::time::{Duration, Instant};
|
|
let agent: ureq::Agent = ureq::Agent::config_builder().http_status_as_error(false).timeout_global(Some(Duration::from_secs(1800))).build().into();
|
|
let mut body = json!({ "messages": messages, "temperature": 0, "max_tokens": max_tokens, "stream": false });
|
|
if let Some(n) = name {
|
|
body["model"] = json!(n);
|
|
}
|
|
let t0 = Instant::now();
|
|
let mut resp = agent
|
|
.post(format!("{}/chat/completions", endpoint.trim_end_matches('/')))
|
|
.header("user-agent", concat!("wai-quantum/", env!("CARGO_PKG_VERSION")))
|
|
.send_json(&body)
|
|
.map_err(|e| format!("{endpoint}: {e}"))?;
|
|
let code = resp.status().as_u16();
|
|
let v: Value = resp.body_mut().read_json().map_err(|e| format!("the model server's reply did not read: {e}"))?;
|
|
if code >= 400 {
|
|
return Err(format!("the model server answered {code}: {v}"));
|
|
}
|
|
let text = v["choices"][0]["message"]["content"].as_str().ok_or_else(|| format!("the model gave no answer: {v}"))?.to_string();
|
|
Ok(Asked { text, usage: v["usage"].clone(), seconds: t0.elapsed().as_secs_f64() })
|
|
}
|
|
|
|
#[cfg(not(feature = "service"))]
|
|
fn ask(_: &str, _: Option<&str>, _: &[Value], _: u32) -> Result<Asked, String> {
|
|
Err("this build has no HTTP client (feature service): run with --scripted".into())
|
|
}
|
|
|
|
/// The model server's energy, read now, in joules, where this host meters it.
|
|
fn meter(pid: Option<u32>) -> Option<f64> {
|
|
pid.and_then(wai_quantum::engine::energy::running).map(|r| r.joules())
|
|
}
|
|
|
|
/// Every task asked and scored: the report.
|
|
pub fn run(a: &RunArgs) -> Result<Value, String> {
|
|
let tasks: Vec<Value> = load_tasks(&a.tasks)?.into_iter().filter(|t| a.only.is_empty() || a.only.iter().any(|n| t["name"] == n.as_str())).collect();
|
|
if tasks.is_empty() {
|
|
return Err("no task to run".into());
|
|
}
|
|
let scripted: Option<Value> = match &a.scripted {
|
|
Some(p) => Some(serde_json::from_str(&std::fs::read_to_string(p).map_err(|e| format!("{}: {e}", p.display()))?).map_err(|e| format!("{}: not JSON: {e}", p.display()))?),
|
|
None => None,
|
|
};
|
|
if let Some(url) = &a.model
|
|
&& !private(url)
|
|
&& !a.allow_remote
|
|
{
|
|
return Err(format!("{url} is neither on this machine nor on a private network: the tasks would go there, and a service may charge for each call; --allow-remote sends them"));
|
|
}
|
|
let metered = a.energy_pid.is_some() && meter(a.energy_pid).is_some();
|
|
let system = system_prompt(a.format);
|
|
let mut results = Vec::new();
|
|
for t in &tasks {
|
|
let name = t["name"].as_str().unwrap_or("");
|
|
let before = if metered { meter(a.energy_pid) } else { None };
|
|
let mut messages = vec![json!({ "role": "system", "content": system }), json!({ "role": "user", "content": t["ask"] })];
|
|
let mut first: Option<Value> = None;
|
|
let mut rounds = 0;
|
|
let mut tokens: Vec<Value> = Vec::new();
|
|
let mut seconds = 0.0;
|
|
let mut r = loop {
|
|
let asked = match (&scripted, &a.model) {
|
|
(Some(s), _) => Ok(Asked { text: s[name].as_str().unwrap_or("").to_string(), usage: Value::Null, seconds: 0.0 }),
|
|
(None, Some(url)) => ask(url, a.name.as_deref(), &messages, a.max_tokens),
|
|
(None, None) => Err("give --model or --scripted".into()),
|
|
};
|
|
let x = match asked {
|
|
Ok(x) => x,
|
|
Err(e) => break json!({ "valid": false, "solved": false, "level": null, "holds": null, "codes": [], "why": format!("the model was not asked: {e}") }),
|
|
};
|
|
seconds += x.seconds;
|
|
tokens.push(x.usage.clone());
|
|
let mut r = score(t, &x.text, a.format);
|
|
r["answer"] = json!(x.text);
|
|
first.get_or_insert_with(|| json!({ "valid": r["valid"], "solved": r["solved"], "level": r["level"] }));
|
|
let wrong = errors(&x.text, a.format);
|
|
if wrong.is_empty() || rounds >= a.repair_rounds || scripted.is_some() {
|
|
break r;
|
|
}
|
|
// Its errors back to the model, as the engine gives them, for a corrected answer.
|
|
rounds += 1;
|
|
messages.push(json!({ "role": "assistant", "content": x.text }));
|
|
messages.push(json!({ "role": "user", "content": format!("The engine could not read your answer:\n{}\nGive the corrected whole answer.", wrong.iter().map(|e| format!("- {e}")).collect::<Vec<_>>().join("\n")) }));
|
|
};
|
|
let after = if metered { meter(a.energy_pid) } else { None };
|
|
r["tokens"] = Value::Array(tokens);
|
|
r["seconds"] = json!(seconds);
|
|
r["repair_rounds"] = json!(rounds);
|
|
r["first_try"] = first.unwrap_or(Value::Null);
|
|
r["joules"] = match (before, after) {
|
|
(Some(b), Some(f)) => json!(f - b),
|
|
_ => Value::Null,
|
|
};
|
|
let mut row = json!({ "name": name });
|
|
if let (Some(o), Some(x)) = (row.as_object_mut(), r.as_object()) {
|
|
o.extend(x.clone());
|
|
}
|
|
eprintln!("{name}: {}{}", if row["solved"] == true { "solved" } else if row["valid"] == true { "valid, not solved" } else { "not valid" }, row["level"].as_str().map(|l| format!(" ({l})")).unwrap_or_default());
|
|
results.push(row);
|
|
}
|
|
let count = |f: &dyn Fn(&Value) -> bool| results.iter().filter(|r| f(r)).count();
|
|
let solved = count(&|r| r["solved"] == true);
|
|
let mut by_level = serde_json::Map::new();
|
|
for r in &results {
|
|
let l = r["level"].as_str().unwrap_or("not asked").to_string();
|
|
let n = by_level.get(&l).and_then(Value::as_u64).unwrap_or(0) + 1;
|
|
by_level.insert(l, json!(n));
|
|
}
|
|
let joules = if metered {
|
|
let total: f64 = results.iter().filter_map(|r| r["joules"].as_f64()).sum();
|
|
json!({
|
|
"total": total,
|
|
"per_solved": if solved > 0 { json!(total / solved as f64) } else { Value::Null },
|
|
"covers": if cfg!(target_os = "macos") { "the model server's CPU energy over each call, as the kernel counts it for its process (its own threads, plus what system services billed to it); the GPU, memory and the rest of the machine are not counted" } else { "the model server's process energy as this host reads it" },
|
|
})
|
|
} else {
|
|
json!({ "unmetered": match (a.energy_pid, &a.scripted) {
|
|
(_, Some(_)) => "a scripted model spends nothing worth metering",
|
|
(None, _) => "give --energy-pid with the model server's process to meter it",
|
|
(Some(_), _) => "this host has no per-process energy counter the harness reads (macOS has one)",
|
|
} })
|
|
};
|
|
let seconds: f64 = results.iter().filter_map(|r| r["seconds"].as_f64()).sum();
|
|
Ok(json!({
|
|
"format": REPORT,
|
|
"about": "One model on one task set, its answers scored by the engine: valid (the program reads), the verdict's level and whether the intent holds, solved (it holds by proof or evidence), and joules per solved task where the model server's energy is metered. A report ranks nothing.",
|
|
"model": match (&a.scripted, &a.model) {
|
|
(Some(p), _) => json!({ "scripted": p.file_name().map(|f| f.to_string_lossy().into_owned()) }),
|
|
(None, url) => json!({ "server": url, "name": a.name }),
|
|
},
|
|
"answer_format": a.format.name(),
|
|
"tasks": results,
|
|
"summary": { "tasks": results.len(), "valid": count(&|r| r["valid"] == true), "solved": solved, "first_try": { "valid": count(&|r| r["first_try"]["valid"] == true), "solved": count(&|r| r["first_try"]["solved"] == true) }, "repair_rounds": a.repair_rounds, "by_level": by_level, "seconds": seconds, "joules": joules },
|
|
}))
|
|
}
|
|
|
|
/// One intent as the answer view layer writes it: the reference program it
|
|
/// names (its block's name and text), the `@Check`'s intent argument, and its
|
|
/// tolerance.
|
|
type LayerIntent = (Option<(String, String)>, String, Option<f64>);
|
|
|
|
/// A task's intents as the answer view layer writes them.
|
|
fn layer_intents(task: &Value) -> Vec<LayerIntent> {
|
|
let one = |i: &Value, k: usize| -> LayerIntent {
|
|
let tol = i["tolerance"].as_f64();
|
|
for kind in ["equivalent", "prepares"] {
|
|
if let Some(src) = i[kind].as_str() {
|
|
let name = format!("reference{}", if k == 0 { String::new() } else { format!("_{}", k + 1) });
|
|
return (Some((name.clone(), src.to_string())), format!("{{\"{kind}\": {name}}}"), tol);
|
|
}
|
|
}
|
|
(None, serde_json::to_string(&i["distribution"]).unwrap_or_default().replace("\":", "\": ").replace(",\"", ", \""), tol)
|
|
};
|
|
match &task["intent"] {
|
|
Value::Array(list) => list.iter().enumerate().map(|(k, i)| one(i, k)).collect(),
|
|
i => vec![one(i, 0)],
|
|
}
|
|
}
|
|
|
|
/// One task's answer in the answer view layer: the program, its circuit, a
|
|
/// sample when it measures, a verdict for each intent, and a seal; or, not
|
|
/// `full`, the program with the sample and the verdicts alone, the same
|
|
/// request a job's JSON carries. Noise that more than one call runs under is
|
|
/// declared once, as a value, and passed by its name, as a job states it
|
|
/// once.
|
|
pub fn layer_answer(task: &Value, full: bool) -> String {
|
|
let qasm = task["qasm"].as_str().unwrap_or("");
|
|
let measures = wai_quantum::qasm::parse(qasm).map(|p| !p.measured.is_empty()).unwrap_or(false);
|
|
let run = &task["run"];
|
|
let shots = run["shots"].as_u64().unwrap_or(100);
|
|
let intents = layer_intents(task);
|
|
let written = run.get("noise").map(|n| serde_json::to_string(n).unwrap_or_default().replace("\":", "\": ").replace(",\"", ", \""));
|
|
let shared = written.is_some() && usize::from(measures) + intents.len() > 1;
|
|
let mut views: Vec<String> = if full { vec!["circuit".into()] } else { Vec::new() };
|
|
let mut lines: Vec<String> = if full { vec!["circuit = Circuit(prog)".into()] } else { Vec::new() };
|
|
let noise = if shared {
|
|
lines.push(format!("noise = {}", written.unwrap_or_default()));
|
|
Some("noise".to_string())
|
|
} else {
|
|
written
|
|
};
|
|
if measures {
|
|
views.push("counts".into());
|
|
lines.push(format!("counts = Histogram(@Sample(prog, {shots}{}))", noise.as_ref().map(|n| format!(", null, {n}")).unwrap_or_default()));
|
|
}
|
|
let mut blocks = String::new();
|
|
for (k, (block, intent, tol)) in intents.iter().enumerate() {
|
|
let name = if intents.len() == 1 { "check".to_string() } else { format!("check_{}", k + 1) };
|
|
if let Some((n, src)) = block {
|
|
blocks.push_str(&format!("```qasm {n}\n{src}```\n"));
|
|
}
|
|
let extra = match (tol, &noise) {
|
|
(Some(t), Some(n)) => format!(", {t}, {n}"),
|
|
(Some(t), None) => format!(", {t}"),
|
|
(None, Some(n)) => format!(", null, {n}"),
|
|
(None, None) => String::new(),
|
|
};
|
|
lines.push(format!("{name} = Verdict(@Check(prog, {intent}{extra}))"));
|
|
views.push(name);
|
|
}
|
|
if full {
|
|
let sealed: Vec<&str> = std::iter::once("prog").chain(views.iter().filter(|v| *v != "circuit").map(String::as_str)).collect();
|
|
lines.push(format!("seal = Receipt(@Seal([{}]))", sealed.join(", ")));
|
|
views.push("seal".into());
|
|
}
|
|
format!("root = Answer([{}])\n```qasm prog\n{qasm}```\n{blocks}{}\n", views.join(", "), lines.join("\n"))
|
|
}
|
|
|
|
/// One task's request as a job's JSON: what an agent sends a run tool today.
|
|
/// An intent's name is left out, as the answer view layer has none.
|
|
pub fn job_answer(task: &Value) -> String {
|
|
let mut req = json!({ "qasm": task["qasm"] });
|
|
for (k, v) in task["run"].as_object().into_iter().flatten() {
|
|
req[k] = v.clone();
|
|
}
|
|
let mut intent = task["intent"].clone();
|
|
for i in intent.as_array_mut().into_iter().flatten() {
|
|
if let Some(o) = i.as_object_mut() {
|
|
o.remove("name");
|
|
}
|
|
}
|
|
req["intent"] = intent;
|
|
req.to_string()
|
|
}
|
|
|
|
/// Each task's answer written four ways: the program alone, in a fenced
|
|
/// block; the answer view layer, whole and with the request alone; and a
|
|
/// job's JSON. The same program and the same request in the last two.
|
|
pub fn corpus(tasks: &[Value]) -> Result<Value, String> {
|
|
let mut out = Vec::new();
|
|
for t in tasks {
|
|
let qasm = t["qasm"].as_str().ok_or_else(|| format!("{}: no reference program", t["name"]))?;
|
|
out.push(json!({
|
|
"name": t["name"],
|
|
"plain": format!("```qasm\n{qasm}```\n"),
|
|
"views": layer_answer(t, true),
|
|
"views_request": layer_answer(t, false),
|
|
"job": job_answer(t),
|
|
}));
|
|
}
|
|
Ok(json!({
|
|
"format": ANSWERS,
|
|
"about": "Each task's answer written four ways, for measuring what each costs a model in tokens: plain, the program alone in a fenced block (what to compute and show is not in it); views, the answer view layer (the program, its circuit, a sample when it measures, a verdict for each intent and a seal); views_request, the layer with the sample and the verdicts alone; job, the request an agent sends a run tool as JSON (the program as an escaped string, the run and the intents). views_request and job carry the same program and the same request. Written by wai-quantum eval corpus; a test holds it to the task set.",
|
|
"answers": out,
|
|
}))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn tasks_path() -> PathBuf {
|
|
Path::new(env!("CARGO_MANIFEST_DIR")).join("../quantum-conformance/eval/tasks.json")
|
|
}
|
|
|
|
fn tasks() -> Vec<Value> {
|
|
load_tasks(&tasks_path()).unwrap()
|
|
}
|
|
|
|
/// Ground truth first: each task's reference program is solved by proof
|
|
/// or evidence, and a program that does nothing on its qubits is not.
|
|
#[test]
|
|
fn every_task_s_reference_is_solved_and_an_empty_program_is_not() {
|
|
let ts = tasks();
|
|
assert!(ts.len() >= 12, "{}", ts.len());
|
|
for t in &ts {
|
|
let name = t["name"].as_str().unwrap();
|
|
let reference = format!("```qasm\n{}```\n", t["qasm"].as_str().unwrap());
|
|
let s = score(t, &reference, Format::Qasm);
|
|
assert_eq!((s["valid"].as_bool(), s["solved"].as_bool()), (Some(true), Some(true)), "{name}: {s}");
|
|
assert!(t["expect"]["levels"].as_array().unwrap().iter().any(|l| l == &s["level"]), "{name}: {s}");
|
|
let n = t["qubits"].as_u64().unwrap();
|
|
let empty = format!("OPENQASM 3.0;\nqubit[{n}] q;\nbit[{n}] c;\nc = measure q;\n");
|
|
let e = score(t, &empty, Format::Qasm);
|
|
assert_eq!(e["solved"], false, "{name}: an empty program must not solve it: {e}");
|
|
}
|
|
// Every task names where it comes from: an intent case or an example, or says it is new.
|
|
let intents: Value = serde_json::from_str(&std::fs::read_to_string(tasks_path().parent().unwrap().join("../intents/manifest.json")).unwrap()).unwrap();
|
|
let cases: Vec<&str> = intents["cases"].as_array().unwrap().iter().map(|c| c["name"].as_str().unwrap()).collect();
|
|
let examples: Vec<&str> = guide::examples().iter().map(|e| e.name).collect();
|
|
for t in &ts {
|
|
for f in t["from"].as_array().unwrap().iter().map(|f| f.as_str().unwrap()) {
|
|
if let Some(rest) = f.strip_prefix("examples: ") {
|
|
assert!(examples.contains(&rest), "{f}");
|
|
} else if let Some(rest) = f.strip_prefix("intents: ") {
|
|
let first = rest.split([',', ' ']).next().unwrap();
|
|
assert!(cases.contains(&first) || rest.starts_with("the same circuit"), "{f}");
|
|
} else {
|
|
assert!(f.starts_with("new: "), "{f}");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn an_answer_is_read_for_its_program_and_scored() {
|
|
let t = &tasks().into_iter().find(|t| t["name"] == "bell").unwrap();
|
|
let bell = t["qasm"].as_str().unwrap();
|
|
// Thinking aloud, prose and the program: the program is what is scored.
|
|
let s = score(t, &format!("<think>a Bell pair: H then CX</think>Here it is:\n```openqasm\n{bell}```\nIt gives 00 and 11."), Format::Qasm);
|
|
assert_eq!((s["solved"].as_bool(), s["level"].as_str()), (Some(true), Some("proof")), "{s}");
|
|
let bare = score(t, bell, Format::Qasm);
|
|
assert_eq!(bare["solved"], true, "a bare program reads too: {bare}");
|
|
let none = score(t, "I cannot help with that.", Format::Qasm);
|
|
assert_eq!((none["valid"].as_bool(), none["why"].as_str()), (Some(false), Some("the answer holds no program")));
|
|
let bad = score(t, "```qasm\nOPENQASM 3.0;\nqubit[2] q;\nfoo q[0];\n```", Format::Qasm);
|
|
assert_eq!((bad["valid"].as_bool(), bad["level"].as_str(), bad["codes"][0].as_str()), (Some(false), Some("refused"), Some("Q103")), "{bad}");
|
|
// A program that measures its qubits in another order prepares the same state, read alike.
|
|
let ghz = tasks().into_iter().find(|t| t["name"] == "ghz-4-prepares").unwrap();
|
|
let backwards = "```qasm\nOPENQASM 3.0;\nqubit[4] q;\nbit[4] c;\nh q[0];\ncx q[0], q[1];\ncx q[0], q[2];\ncx q[0], q[3];\nc[3] = measure q[3];\nc[2] = measure q[2];\nc[1] = measure q[1];\nc[0] = measure q[0];\n```";
|
|
assert_eq!(score(&ghz, backwards, Format::Qasm)["solved"], true);
|
|
let wrong = score(t, "```qasm\nOPENQASM 3.0;\nqubit[2] q;\nbit[2] c;\nh q[0];\nc = measure q;\n```", Format::Qasm);
|
|
assert_eq!((wrong["valid"].as_bool(), wrong["holds"].as_bool(), wrong["solved"].as_bool()), (Some(true), Some(false), Some(false)), "{wrong}");
|
|
// In the answer view layer: a number the model wrote is an error, and the answer is not valid.
|
|
let views = layer_answer(t, true);
|
|
assert_eq!(score(t, &views, Format::Views)["solved"], true);
|
|
let made_up = score(t, &format!("{views}counts = Histogram(512)\n"), Format::Views);
|
|
assert_eq!((made_up["valid"].as_bool(), made_up["solved"].as_bool()), (Some(false), Some(false)), "{made_up}");
|
|
assert!(made_up["codes"].as_array().unwrap().iter().any(|c| c == "J104"), "{made_up}");
|
|
}
|
|
|
|
/// The harness end to end with a scripted model: the summary counts what
|
|
/// the engine found, and a scripted model is not metered.
|
|
#[test]
|
|
fn a_scripted_run_reports_what_the_engine_found() {
|
|
let dir = std::env::temp_dir().join(format!("wai-quantum-eval-{}", std::process::id()));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
let ts = tasks();
|
|
let mut answers = serde_json::Map::new();
|
|
for (i, t) in ts.iter().enumerate() {
|
|
let a = match i % 3 {
|
|
0 => format!("```qasm\n{}```", t["qasm"].as_str().unwrap()),
|
|
1 => "```qasm\nOPENQASM 3.0;\nqubit[1] q;\nfoo q[0];\n```".to_string(),
|
|
_ => String::new(),
|
|
};
|
|
answers.insert(t["name"].as_str().unwrap().to_string(), json!(a));
|
|
}
|
|
let scripted = dir.join("answers.json");
|
|
std::fs::write(&scripted, Value::Object(answers).to_string()).unwrap();
|
|
let args = RunArgs { tasks: tasks_path(), model: None, name: None, scripted: Some(scripted), format: Format::Qasm, energy_pid: None, allow_remote: false, max_tokens: 64, repair_rounds: 0, only: vec![], out: None };
|
|
let r = run(&args).unwrap();
|
|
let s = &r["summary"];
|
|
let third = ts.len().div_ceil(3);
|
|
assert_eq!((s["tasks"].as_u64(), s["solved"].as_u64(), s["valid"].as_u64()), (Some(ts.len() as u64), Some(third as u64), Some(third as u64)), "{s}");
|
|
assert!(s["joules"]["unmetered"].as_str().unwrap().contains("scripted"), "{s}");
|
|
assert_eq!(r["format"], REPORT);
|
|
let errs = crate::mcp::schemas().check("eval-report.v1.json", &r);
|
|
assert!(errs.is_empty(), "the report does not follow its schema: {}\n{r}", errs.join("; "));
|
|
std::fs::remove_dir_all(&dir).ok();
|
|
}
|
|
|
|
/// A model server on this machine, answering as a chat-completions
|
|
/// server does; one elsewhere is refused unless allowed.
|
|
#[cfg(feature = "service")]
|
|
#[test]
|
|
fn a_model_on_this_machine_is_asked_and_one_elsewhere_is_refused() {
|
|
use std::io::{BufRead, BufReader, Read, Write};
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
let asked = std::sync::Arc::new(std::sync::Mutex::new(Vec::<Value>::new()));
|
|
let seen = asked.clone();
|
|
std::thread::spawn(move || {
|
|
for stream in listener.incoming().flatten() {
|
|
let mut r = BufReader::new(stream.try_clone().unwrap());
|
|
let mut len = 0usize;
|
|
loop {
|
|
let mut line = String::new();
|
|
if r.read_line(&mut line).unwrap_or(0) == 0 || line == "\r\n" {
|
|
break;
|
|
}
|
|
if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") {
|
|
len = v.trim().parse().unwrap_or(0);
|
|
}
|
|
}
|
|
let mut body = vec![0; len];
|
|
r.read_exact(&mut body).unwrap();
|
|
let req: Value = serde_json::from_slice(&body).unwrap();
|
|
let task = req["messages"][1]["content"].as_str().unwrap_or("").to_string();
|
|
let repairing = req["messages"].as_array().map_or(0, Vec::len) > 2;
|
|
seen.lock().unwrap().push(req);
|
|
// The Bell pair at once; the superposition first with a missing `;`, then repaired.
|
|
let content = if task.contains("Bell pair") {
|
|
"```qasm\nOPENQASM 3.0;\nqubit[2] q;\nbit[2] c;\nh q[0];\ncx q[0], q[1];\nc = measure q;\n```"
|
|
} else if repairing {
|
|
"```qasm\nOPENQASM 3.0;\nqubit[1] q;\nbit[1] c;\nh q[0];\nc = measure q;\n```"
|
|
} else {
|
|
"```qasm\nOPENQASM 3.0;\nqubit[1] q\nbit[1] c;\nh q[0];\nc = measure q;\n```"
|
|
};
|
|
let reply = json!({ "choices": [{ "message": { "role": "assistant", "content": content } }], "usage": { "prompt_tokens": 10, "completion_tokens": 5 } }).to_string();
|
|
let mut s = stream;
|
|
let _ = write!(s, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{reply}", reply.len());
|
|
}
|
|
});
|
|
let url = format!("http://127.0.0.1:{port}/v1");
|
|
let args = RunArgs { tasks: tasks_path(), model: Some(url), name: Some("scripted-server".into()), scripted: None, format: Format::Qasm, energy_pid: None, allow_remote: false, max_tokens: 64, repair_rounds: 1, only: vec!["bell".into(), "plus".into()], out: None };
|
|
let r = run(&args).unwrap();
|
|
// The superposition's missing `;` sent back with its code, and repaired: solved on the second round.
|
|
assert_eq!((r["summary"]["tasks"].as_u64(), r["summary"]["solved"].as_u64(), r["summary"]["first_try"]["solved"].as_u64()), (Some(2), Some(2), Some(1)), "{r}");
|
|
assert_eq!((r["tasks"][0]["repair_rounds"].as_u64(), r["tasks"][1]["repair_rounds"].as_u64()), (Some(1), Some(0)), "{r}");
|
|
assert_eq!(r["tasks"][1]["tokens"][0]["completion_tokens"], 5);
|
|
let errs = crate::mcp::schemas().check("eval-report.v1.json", &r);
|
|
assert!(errs.is_empty(), "the report does not follow its schema: {}\n{r}", errs.join("; "));
|
|
let reqs = asked.lock().unwrap();
|
|
assert!(reqs.iter().all(|q| q["temperature"] == 0 && q["model"] == "scripted-server" && q["messages"][0]["content"].as_str().unwrap().contains("```qasm block")), "{reqs:?}");
|
|
let repair = reqs.iter().find(|q| q["messages"].as_array().unwrap().len() == 4).expect("a repair round");
|
|
assert!(repair["messages"][3]["content"].as_str().unwrap().contains("[Q004]"), "the error went back with its code: {repair}");
|
|
assert!(reqs.iter().all(|q| q.get("api_key").is_none()), "no key is sent");
|
|
let far = RunArgs { model: Some("https://models.example.net/v1".into()), ..args };
|
|
assert!(run(&far).unwrap_err().contains("--allow-remote"));
|
|
assert!(private("http://127.0.0.1:8080/v1") && private("http://192.168.4.47:8080") && private("http://100.100.141.123:8080/v1") && private("http://localhost/v1") && private("http://[::1]:8080"));
|
|
assert!(!private("https://api.example.com/v1") && !private("http://8.8.8.8/v1"));
|
|
}
|
|
|
|
/// The corpus is what `eval corpus` writes from the task set; each
|
|
/// answer in the layer reads with no error and runs to its intents.
|
|
#[test]
|
|
fn the_corpus_is_the_task_set_written_three_ways() {
|
|
let ts = tasks();
|
|
let c = corpus(&ts).unwrap();
|
|
for (t, a) in ts.iter().zip(c["answers"].as_array().unwrap()) {
|
|
for form in ["views", "views_request"] {
|
|
let views = a[form].as_str().unwrap();
|
|
let parsed = answer::parse(views);
|
|
assert!(parsed.errors().is_empty() && parsed.diags.is_empty(), "{} {form}: {:?}\n{views}", t["name"], parsed.diags);
|
|
assert_eq!(score(t, views, Format::Views)["solved"], true, "{} {form}", t["name"]);
|
|
}
|
|
let job: Value = serde_json::from_str(a["job"].as_str().unwrap()).unwrap();
|
|
if let Some(noise) = job["noise"].as_object() {
|
|
// Noise the sample and the verdict share is written once, as the job writes it.
|
|
let channel = noise.keys().next().unwrap();
|
|
assert_eq!(a["views_request"].as_str().unwrap().matches(channel.as_str()).count(), 1, "{}", t["name"]);
|
|
}
|
|
assert_eq!(&job["qasm"], &t["qasm"]);
|
|
assert_eq!(job["intent"].as_array().map(Vec::len), t["intent"].as_array().map(Vec::len));
|
|
assert!(job["intent"].as_array().unwrap().iter().all(|i| i.get("name").is_none()));
|
|
}
|
|
// The corpus, and the token counts measured on it, follow their schemas.
|
|
let errs = crate::mcp::schemas().check("eval-answers.v1.json", &c);
|
|
assert!(errs.is_empty(), "the corpus does not follow its schema: {}", errs.join("; "));
|
|
let cost: Value = serde_json::from_str(&std::fs::read_to_string(tasks_path().parent().unwrap().join("token-cost.json")).unwrap()).unwrap();
|
|
let errs = crate::mcp::schemas().check("token-cost.v1.json", &cost);
|
|
assert!(errs.is_empty(), "token-cost.json does not follow its schema: {}", errs.join("; "));
|
|
let path = tasks_path().parent().unwrap().join("answers.json");
|
|
let want = serde_json::to_string_pretty(&c).unwrap() + "\n";
|
|
if std::env::var_os("WAI_QUANTUM_BLESS").is_some() {
|
|
std::fs::write(&path, &want).unwrap();
|
|
return;
|
|
}
|
|
assert!(std::fs::read_to_string(&path).unwrap() == want, "{} is not what eval corpus writes: run the tests with WAI_QUANTUM_BLESS=1", path.display());
|
|
}
|
|
}
|