open-standards/wai/wai-quantum/src/engine/study.rs

//! Measuring whether the co-design works: a study people take part in, with
//! every measure computed by the engine and none by a model.
//!
//! - **A protocol** ([`Protocol`]), written and sealed before any data: the
//!   hypotheses, the primary measure, the conditions (each a setting of the
//!   Studio's [`FLAGS`]) and the analysis (resamples, seed, level).
//! - **Assignment** ([`assign`]): a participant's condition is the protocol's
//!   digest and the participant's key hashed together, so it is fixed before
//!   they begin and anyone can check it.
//! - **A record** ([`Record`]): what one participant made and did, sealed
//!   with their key: the program's history, their predictions, their choices
//!   among candidates, the probe tasks they did without the assistant, and
//!   their answers on ownership.
//! - **The measures** ([`measure`]), the report's eight: quality (engine
//!   measures of the edits the engine proves equal to what they replaced) and
//!   its trend over time; diversity (the Vendi score of the participant's
//!   designs); novelty (distance from the known constructions); selection
//!   skill (how often they took the engine's best of the candidates shown);
//!   calibration (their predictions against the runs); unassisted skill (the
//!   probes, scored again by the engine); ownership (their answers, beside
//!   their share of the gates by the history); and convergence on a model's
//!   proposals.
//! - **The cohort** ([`cohort`]): every record checked against the protocol
//!   (its digest, its seal, its condition) and refused with the reason if it
//!   fails; per condition the mean of each measure; each condition's
//!   difference from the control on the primary measure, with a seeded
//!   bootstrap interval; Cronbach's alpha of the ownership items; and the
//!   Vendi score of each condition's final designs.
//!
//! Nothing here is sent anywhere: a participant gives their record to whoever
//! runs the study, and the report is computed where it is run.

use super::converge::{self, Design};
use super::discover::{Evaluator, Spec};
use super::versions::{Hand, History, Kind};
use super::{critique, equiv, novelty, receipt};
use crate::qasm::{self, Inputs};
use ed25519_dalek::SigningKey;
use serde_json::{Value, json};
use std::collections::{BTreeMap, BTreeSet};

/// The form of a protocol and a record this engine reads and writes.
pub const FORM: u32 = 1;

/// The Studio's features a condition sets, each on or off, with what it is.
pub const FLAGS: &[(&str, &str)] = &[
    ("critique", "the engine's critique on the live diagram"),
    ("assistant_asks", "the assistant asks before it proposes, and speaks only when asked"),
    ("alternatives", "Alternatives: improve and explore, laid out on the engine's axes"),
    ("predict_first", "predict before a run"),
    ("workbench", "the Workbench: a family searched, every proposal judged"),
    ("rules", "the person's rules for proposals"),
    ("shortcuts", "the classical-shortcut lens"),
    ("novelty", "what is new: labels, Found"),
    ("taste", "the taste profile: comparisons, ranking"),
    ("rewrites", "the rewrite lens"),
    ("convergence", "your own way: convergence and exploring on your own"),
];

/// The measures a protocol may name as its primary one.
pub const MEASURES: &[(&str, &str)] = &[
    ("quality", "the share of the participant's edits, proved equal to what they replaced, that made it cheaper"),
    ("quality_trend", "how that share moved over their history (later half less earlier half)"),
    ("selection", "how often they took the engine's best of the candidates shown"),
    ("calibration_off", "how far their predicted share was from the run's, on average"),
    ("calibration_slope", "the slope of the run's share on their prediction (1 is calibrated)"),
    ("unassisted", "their mean probe score, done without the assistant (1 matches the reference)"),
    ("ownership", "their mean answer on the ownership items, 1 to 7"),
    ("authorship", "their share of the program's gates, by the history"),
    ("novelty", "the mean distance of their designs from the known constructions"),
    ("diversity", "the Vendi score of their distinct designs"),
    ("convergence", "a model's share of their program's gates"),
];

/// The ownership items, each answered from 1 (not at all) to 7 (completely).
pub const OWNERSHIP: &[&str] = &[
    "This program is mine.",
    "I decided how it is built.",
    "I could explain every part of it.",
    "I would put my name to it.",
    "I made it more than any tool did.",
];

/// The probe tasks, done without the assistant and scored by the engine:
/// `(id, what to do, the specification it is judged by)`. A score is the
/// reference's cost over the answer's, so 1 matches the reference and more
/// beats it; an answer the engine does not verify scores 0.
pub fn probes() -> Vec<(&'static str, &'static str, Value)> {
    vec![
        (
            "ghz4",
            "Prepare (|0000⟩ + |1111⟩)/√2 on four qubits with h and cx, as shallow as you can.",
            json!({ "name": "probe ghz4", "task": "prepares", "reference": "OPENQASM 3.0;\nqubit[4] q;\nh q[0];\ncx q[0], q[1];\ncx q[1], q[2];\ncx q[2], q[3];\n", "inputs": {}, "gates": ["h", "cx"], "cost": { "depth": 1.0, "two_qubit": 0.1 }, "axes": ["two_qubit", "depth"] }),
        ),
        (
            "bell_cz",
            "Prepare the Bell state (|00⟩ + |11⟩)/√2 using only h and cz.",
            json!({ "name": "probe bell_cz", "task": "prepares", "reference": "OPENQASM 3.0;\nqubit[2] q;\nh q[0];\nh q[1];\ncz q[0], q[1];\nh q[1];\n", "inputs": {}, "gates": ["h", "cz"], "cost": { "gates": 1.0 }, "axes": ["two_qubit", "depth"] }),
        ),
        (
            "swap",
            "Write a circuit equal to swap q[0], q[1] using only cx.",
            json!({ "name": "probe swap", "task": "equivalent", "reference": "OPENQASM 3.0;\nqubit[2] q;\ncx q[0], q[1];\ncx q[1], q[0];\ncx q[0], q[1];\n", "inputs": {}, "gates": ["cx"], "cost": { "gates": 1.0 }, "axes": ["two_qubit", "depth"] }),
        ),
        (
            "tidy",
            "Write the same circuit as h q[0]; x q[1]; x q[1]; cx q[0], q[1]; rz(0.25) q[0]; rz(0.5) q[0]; with as few gates as you can.",
            json!({ "name": "probe tidy", "task": "equivalent", "reference": "OPENQASM 3.0;\nqubit[2] q;\nh q[0];\nx q[1];\nx q[1];\ncx q[0], q[1];\nrz(0.25) q[0];\nrz(0.5) q[0];\n", "inputs": {}, "cost": { "gates": 1.0 }, "axes": ["two_qubit", "depth"] }),
        ),
    ]
}

/// The engine's score of `source` as an answer to the probe `id`.
pub fn score_probe(id: &str, source: &str) -> Result<Value, String> {
    let (_, what, spec) = probes().into_iter().find(|(p, _, _)| *p == id).ok_or_else(|| format!("{id}: no such probe"))?;
    let ev = Evaluator::new(Spec::from_json(&spec)?)?;
    let reference = ev.evaluate(spec["reference"].as_str().unwrap_or_default());
    let answer = ev.evaluate(source);
    let rc = reference.cost.ok_or("the probe's reference does not verify")?;
    let score = match (answer.verified, answer.cost) {
        (true, Some(c)) if c > 0.0 => rc / c,
        (true, Some(_)) => 1.0,
        _ => 0.0,
    };
    Ok(json!({ "probe": id, "task": what, "verified": answer.verified, "cost": answer.cost, "reference_cost": rc, "score": round(score), "failure": answer.failure }))
}

/// A study, before any data.
#[derive(Clone, Debug, PartialEq)]
pub struct Protocol {
    pub value: Value,
}

fn round(x: f64) -> f64 {
    (x * 1000.0).round() / 1000.0
}

impl Protocol {
    /// A protocol from its JSON, checked: a study name, at least one
    /// hypothesis, two conditions or more (each with a name and only known
    /// flags), a control among them, a primary measure the engine knows, and
    /// an analysis with resamples, a seed and a level.
    pub fn from_json(v: &Value) -> Result<Protocol, String> {
        let need = |k: &str| v.get(k).ok_or_else(|| format!("the protocol needs \"{k}\""));
        need("study")?.as_str().filter(|s| !s.is_empty()).ok_or("the study needs a name")?;
        if need("hypotheses")?.as_array().is_none_or(|h| h.is_empty() || h.iter().any(|x| !x.is_string())) {
            return Err("the protocol needs hypotheses: one sentence or more, written before any data".into());
        }
        let primary = need("primary")?.as_str().ok_or("primary: a measure's name")?;
        if !MEASURES.iter().any(|(m, _)| *m == primary) {
            return Err(format!("{primary}: the primary measure is one of {}", MEASURES.iter().map(|(m, _)| *m).collect::<Vec<_>>().join(", ")));
        }
        let conds = need("conditions")?.as_array().ok_or("conditions: a list")?;
        if conds.len() < 2 {
            return Err("a study compares two conditions or more".into());
        }
        let mut names = BTreeSet::new();
        for c in conds {
            let name = c.get("name").and_then(Value::as_str).ok_or("each condition needs a name")?;
            if !names.insert(name) {
                return Err(format!("{name}: two conditions have this name"));
            }
            for (flag, on) in c.get("flags").and_then(Value::as_object).into_iter().flatten() {
                if !FLAGS.iter().any(|(f, _)| f == flag) || !on.is_boolean() {
                    return Err(format!("{name}: {flag} is not a flag (flags: {}), or is not true or false", FLAGS.iter().map(|(f, _)| *f).collect::<Vec<_>>().join(", ")));
                }
            }
        }
        let control = need("control")?.as_str().ok_or("control: a condition's name")?;
        if !names.contains(control) {
            return Err(format!("{control}: the control is not one of the conditions"));
        }
        let a = need("analysis")?;
        let resamples = a.get("resamples").and_then(Value::as_u64).ok_or("analysis.resamples: a whole number")?;
        if !(100..=1_000_000).contains(&resamples) {
            return Err("analysis.resamples: 100 to 1,000,000".into());
        }
        a.get("seed").and_then(Value::as_u64).ok_or("analysis.seed: a whole number")?;
        let level = a.get("level").and_then(Value::as_f64).ok_or("analysis.level: as 0.95")?;
        if !(0.5..1.0).contains(&level) {
            return Err("analysis.level: between 0.5 and 1".into());
        }
        Ok(Protocol { value: v.clone() })
    }

    fn conditions(&self) -> Vec<&Value> {
        self.value["conditions"].as_array().map(|c| c.iter().collect()).unwrap_or_default()
    }
}

fn protocol_request() -> Value {
    json!({ "study_protocol": FORM })
}

/// A protocol sealed with the researcher's key, before any data: `{protocol,
/// seal}`. Its digest names the study in every record.
pub fn seal_protocol(p: &Protocol, key: &SigningKey) -> Value {
    json!({ "protocol": p.value, "seal": receipt::seal(key, &protocol_request(), &p.value) })
}

/// A sealed protocol checked: the protocol and its digest.
pub fn open_protocol(sealed: &Value) -> Result<(Protocol, String), String> {
    let p = Protocol::from_json(sealed.get("protocol").ok_or("a sealed protocol is {protocol, seal}")?)?;
    let seal = sealed.get("seal").ok_or("the protocol is not sealed")?;
    if !receipt::verify(&protocol_request(), &p.value, seal) {
        return Err("the protocol's seal does not verify: it changed after it was sealed".into());
    }
    Ok((p, seal["digest"].as_str().unwrap_or_default().to_string()))
}

/// The condition `participant` (a did:key) is assigned in the study whose
/// sealed protocol has `digest`: fixed by the two, before any data.
pub fn assign(p: &Protocol, digest: &str, participant: &str) -> Value {
    let h = blake3::hash(format!("{digest}|{participant}").as_bytes());
    let n = p.conditions().len() as u64;
    let i = (u64::from_le_bytes(h.as_bytes()[..8].try_into().expect("8 bytes")) % n.max(1)) as usize;
    let c = p.conditions()[i];
    json!({ "condition": c["name"], "flags": c.get("flags").cloned().unwrap_or(json!({})) })
}

fn record_request(digest: &str, participant: &str) -> Value {
    json!({ "study_record": digest, "participant": participant })
}

/// A participant's record sealed with their key: `{record, seal}`. The
/// record names the protocol's digest, the participant (the key's did) and
/// their condition, and holds `history`, `predictions`, `choices`, `probes`
/// and `ownership` as the Studio keeps them.
pub fn seal_record(record: &Value, key: &SigningKey) -> Result<Value, String> {
    let did = crate::quantum_receipt::did_key(&key.verifying_key());
    let mut r = record.clone();
    r["record"] = json!(FORM);
    r["participant"] = json!(did);
    let digest = r.get("protocol").and_then(Value::as_str).ok_or("the record names no protocol")?.to_string();
    Ok(json!({ "record": r, "seal": receipt::seal(key, &record_request(&digest, &did), &r) }))
}

/// A sealed record checked against a protocol: the seal holds, the key is the
/// participant's, the protocol is this one, and the condition is the one
/// assigned. The record, or why it is refused.
pub fn open_record(sealed: &Value, p: &Protocol, digest: &str) -> Result<Value, String> {
    let r = sealed.get("record").ok_or("a sealed record is {record, seal}")?;
    let seal = sealed.get("seal").ok_or("the record is not sealed")?;
    let did = r.get("participant").and_then(Value::as_str).ok_or("the record names no participant")?;
    if r.get("protocol").and_then(Value::as_str) != Some(digest) {
        return Err("it was made for another study".into());
    }
    if !receipt::verify(&record_request(digest, did), r, seal) {
        return Err("its seal does not verify: it changed after it was sealed".into());
    }
    if seal.get("key").and_then(Value::as_str).and_then(super::job::did_of).as_deref() != Some(did) {
        return Err("it is sealed by a key other than its participant's".into());
    }
    let assigned = assign(p, digest, did);
    if r.get("condition") != assigned.get("condition") {
        return Err(format!("its condition is {}, and the participant is assigned {}", r["condition"], assigned["condition"]));
    }
    Ok(r.clone())
}

fn slope(ys: &[f64]) -> Option<f64> {
    let n = ys.len();
    if n < 3 {
        return None;
    }
    let mx = (n - 1) as f64 / 2.0;
    let my = ys.iter().sum::<f64>() / n as f64;
    let num: f64 = ys.iter().enumerate().map(|(i, y)| (i as f64 - mx) * (y - my)).sum();
    let den: f64 = (0..n).map(|i| (i as f64 - mx).powi(2)).sum();
    Some(num / den)
}

fn mean(v: &[f64]) -> Option<f64> {
    (!v.is_empty()).then(|| v.iter().sum::<f64>() / v.len() as f64)
}

/// The engine's measures of one participant, from their record: each one
/// computed again here from what they made and did.
pub fn measure(record: &Value) -> Result<Value, String> {
    let inputs: Inputs = match record.get("inputs") {
        None | Some(Value::Null) => Inputs::new(),
        Some(v) => serde_json::from_value(v.clone()).map_err(|e| format!("inputs: {e}"))?,
    };
    let history = match record.get("history") {
        None | Some(Value::Null) => None,
        Some(v) => Some(History::from_json(v)?),
    };
    let mut out = json!({ "participant": record.get("participant"), "condition": record.get("condition") });

    // Quality: the person's edits the engine proves equal to their parent.
    if let Some(h) = &history {
        let at: BTreeMap<&str, usize> = h.versions.iter().enumerate().map(|(i, v)| (v.id.as_str(), i)).collect();
        let mut scores = Vec::new();
        let mut deltas: BTreeMap<&str, Vec<f64>> = BTreeMap::new();
        for v in h.versions.iter().filter(|v| v.by.kind == Kind::Person) {
            let Some(parent) = v.parents.first().and_then(|p| at.get(p.as_str())).map(|&i| &h.versions[i]) else { continue };
            let (Ok(a), Ok(b)) = (qasm::parse_with(&parent.source, &inputs), qasm::parse_with(&v.source, &inputs)) else { continue };
            let e = equiv::equivalent_programs(&a, &b);
            if e["verdict"] != "equivalent" || e["how"] == "random" {
                continue;
            }
            let (ma, mb) = (critique::measures(&a), critique::measures(&b));
            let keys = ["two_qubit", "depth", "t_count", "gates"];
            let d: Vec<f64> = keys.iter().map(|k| ma[*k].as_f64().unwrap_or(0.0) - mb[*k].as_f64().unwrap_or(0.0)).collect();
            for (k, x) in keys.iter().zip(&d) {
                deltas.entry(*k).or_default().push(*x);
            }
            // Cheaper in one measure and dearer in none is better; the reverse, worse.
            let s = if d.iter().all(|x| *x >= 0.0) && d.iter().any(|x| *x > 0.0) { 1.0 } else if d.iter().all(|x| *x <= 0.0) && d.iter().any(|x| *x < 0.0) { -1.0 } else { 0.0 };
            scores.push(s);
        }
        let improved = scores.iter().filter(|s| **s > 0.0).count();
        let half = scores.len() / 2;
        let trend = (scores.len() >= 4).then(|| mean(&scores[half..]).unwrap_or(0.0) - mean(&scores[..half]).unwrap_or(0.0));
        out["quality"] = json!({
            "edits": scores.len(),
            "improved": improved,
            "worsened": scores.iter().filter(|s| **s < 0.0).count(),
            "share": (!scores.is_empty()).then(|| round(improved as f64 / scores.len() as f64)),
            "trend": trend.map(round),
            "mean_saving": deltas.iter().map(|(k, v)| (k.to_string(), json!(mean(v).map(round)))).collect::<serde_json::Map<_, _>>(),
        });
        // Diversity and novelty of their distinct designs.
        let mut seen = BTreeSet::new();
        let designs: Vec<Design> = h.versions.iter().filter(|v| v.by.kind == Kind::Person && seen.insert(v.source.trim().to_string())).map(|v| Design { name: v.id.clone(), source: v.source.clone(), at: Some(v.at) }).collect();
        let readable: Vec<Design> = designs.into_iter().filter(|d| qasm::parse_with(&d.source, &inputs).is_ok()).collect();
        out["diversity"] = if readable.len() >= 2 { converge::cohort(&readable, &inputs, 0).map(|c| json!({ "designs": readable.len(), "vendi": c["vendi"] }))? } else { json!({ "designs": readable.len(), "vendi": null }) };
        let nov: Vec<f64> = readable.iter().filter_map(|d| novelty::assess(&d.source, &inputs, &[], None, &[])["novelty"].as_f64()).collect();
        out["novelty"] = json!({ "designs": nov.len(), "mean": mean(&nov).map(round) });
        // Authorship, and convergence on a model's proposals.
        if let Ok(head) = h.head()
            && let Ok(a) = h.authorship(&head.id, &inputs)
            && a.of.gates > 0
        {
            let g = |k: Hand| a.gates.get(&k).copied().unwrap_or(0) as f64;
            let person = g(Hand::Person) + 0.5 * (g(Hand::AssistantThenPerson) + g(Hand::AgentThenPerson));
            out["authorship"] = json!(round(person / a.of.gates as f64));
        }
        if let Ok(c) = converge::history(h, &inputs) {
            out["convergence"] = json!({ "share": c["share"], "converging": c["converging"], "organic": c["organic"], "assisted": c["assisted"] });
        }
    }

    // Selection skill: the engine's best of the candidates, taken or not.
    let choices: Vec<&Value> = record.get("choices").and_then(Value::as_array).map(|c| c.iter().collect()).unwrap_or_default();
    let shown: Vec<(u64, u64, u64)> = choices.iter().filter_map(|c| Some((c.get("k")?.as_u64()?, c.get("picked")?.as_u64()?, c.get("best")?.as_u64()?))).filter(|(k, p, b)| *k >= 2 && p < k && b < k).collect();
    out["selection"] = json!({
        "choices": shown.len(),
        "best_taken": shown.iter().filter(|(_, p, b)| p == b).count(),
        "rate": (!shown.is_empty()).then(|| round(shown.iter().filter(|(_, p, b)| p == b).count() as f64 / shown.len() as f64)),
        "chance": mean(&shown.iter().map(|(k, _, _)| 1.0 / *k as f64).collect::<Vec<_>>()).map(round),
    });

    // Calibration: the share they predicted against the run's.
    let preds: Vec<(f64, f64, bool)> = record.get("predictions").and_then(Value::as_array).into_iter().flatten().filter_map(|p| Some((p.get("share")?.as_f64()?, p.get("got")?.as_f64()?, p.get("right").and_then(Value::as_bool).unwrap_or(false)))).filter(|(s, g, _)| (0.0..=1.0).contains(s) && (0.0..=1.0).contains(g)).collect();
    let (xs, ys): (Vec<f64>, Vec<f64>) = preds.iter().map(|(s, g, _)| (*s, *g)).unzip();
    let fit = (xs.len() >= 3).then(|| {
        let (mx, my) = (mean(&xs).unwrap_or(0.0), mean(&ys).unwrap_or(0.0));
        let sxx: f64 = xs.iter().map(|x| (x - mx).powi(2)).sum();
        let sxy: f64 = xs.iter().zip(&ys).map(|(x, y)| (x - mx) * (y - my)).sum();
        (sxx > 1e-12).then(|| (sxy / sxx, my - sxy / sxx * mx))
    }).flatten();
    out["calibration"] = json!({
        "predictions": preds.len(),
        "top_right": preds.iter().filter(|p| p.2).count(),
        "mean_off": mean(&preds.iter().map(|(s, g, _)| (s - g).abs()).collect::<Vec<_>>()).map(round),
        "slope": fit.map(|f| round(f.0)),
        "intercept": fit.map(|f| round(f.1)),
    });

    // Unassisted skill: every probe scored again by the engine.
    let mut probe_scores = Vec::new();
    let mut by_probe: BTreeMap<String, Vec<f64>> = BTreeMap::new();
    for p in record.get("probes").and_then(Value::as_array).into_iter().flatten() {
        let (Some(id), Some(src)) = (p.get("probe").and_then(Value::as_str), p.get("source").and_then(Value::as_str)) else { continue };
        if let Ok(s) = score_probe(id, src) {
            let x = s["score"].as_f64().unwrap_or(0.0);
            probe_scores.push(x);
            by_probe.entry(id.to_string()).or_default().push(x);
        }
    }
    out["unassisted"] = json!({
        "probes": probe_scores.len(),
        "mean": mean(&probe_scores).map(round),
        "trend": slope(&probe_scores).map(round),
        "by_probe": by_probe.iter().map(|(k, v)| (k.clone(), json!(mean(v).map(round)))).collect::<serde_json::Map<_, _>>(),
    });

    // Ownership: the answers, each 1 to 7.
    let answers: Vec<Vec<f64>> = record.get("ownership").and_then(Value::as_array).into_iter().flatten().filter_map(|o| {
        let items: Vec<f64> = o.get("items")?.as_array()?.iter().filter_map(Value::as_f64).collect();
        (items.len() == OWNERSHIP.len() && items.iter().all(|x| (1.0..=7.0).contains(x))).then_some(items)
    }).collect();
    out["ownership"] = json!({
        "answers": answers.len(),
        "mean": mean(&answers.iter().filter_map(|a| mean(a)).collect::<Vec<_>>()).map(round),
        "latest": answers.last().and_then(|a| mean(a)).map(round),
    });
    Ok(out)
}

/// The measures as one number each, by the names in [`MEASURES`], where the
/// record gives one.
pub fn scalars(m: &Value) -> BTreeMap<&'static str, f64> {
    let pick = [
        ("quality", m.pointer("/quality/share")),
        ("quality_trend", m.pointer("/quality/trend")),
        ("selection", m.pointer("/selection/rate")),
        ("calibration_off", m.pointer("/calibration/mean_off")),
        ("calibration_slope", m.pointer("/calibration/slope")),
        ("unassisted", m.pointer("/unassisted/mean")),
        ("ownership", m.pointer("/ownership/mean")),
        ("authorship", m.get("authorship")),
        ("novelty", m.pointer("/novelty/mean")),
        ("diversity", m.pointer("/diversity/vendi")),
        ("convergence", m.pointer("/convergence/share")),
    ];
    pick.into_iter().filter_map(|(k, v)| Some((k, v?.as_f64()?))).collect()
}

/// splitmix64: the bootstrap's resampling, the same from a seed everywhere.
struct Rng(u64);

impl Rng {
    fn next(&mut self) -> u64 {
        self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15);
        let mut z = self.0;
        z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
        z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
        z ^ (z >> 31)
    }
    fn below(&mut self, n: usize) -> usize {
        (self.next() % n as u64) as usize
    }
}

/// The difference of means `b − a` and its bootstrap interval at `level`,
/// resampling each group's participants `resamples` times from `seed`.
fn bootstrap(a: &[f64], b: &[f64], resamples: u64, seed: u64, level: f64) -> Value {
    if a.is_empty() || b.is_empty() {
        return json!({ "difference": null, "low": null, "high": null, "why": "a group has no participant with this measure" });
    }
    let diff = mean(b).unwrap_or(0.0) - mean(a).unwrap_or(0.0);
    let mut rng = Rng(seed);
    let mut ds: Vec<f64> = (0..resamples)
        .map(|_| {
            let ma = (0..a.len()).map(|_| a[rng.below(a.len())]).sum::<f64>() / a.len() as f64;
            let mb = (0..b.len()).map(|_| b[rng.below(b.len())]).sum::<f64>() / b.len() as f64;
            mb - ma
        })
        .collect();
    ds.sort_by(f64::total_cmp);
    let q = |p: f64| ds[((p * (ds.len() - 1) as f64).round() as usize).min(ds.len() - 1)];
    let tail = (1.0 - level) / 2.0;
    json!({ "difference": round(diff), "low": round(q(tail)), "high": round(q(1.0 - tail)), "excludes_zero": q(tail) > 0.0 || q(1.0 - tail) < 0.0 })
}

/// Cronbach's alpha of `responses`, each the same number of items.
fn cronbach(responses: &[Vec<f64>]) -> Option<f64> {
    let k = responses.first()?.len();
    if k < 2 || responses.len() < 2 {
        return None;
    }
    let var = |xs: &[f64]| {
        let m = xs.iter().sum::<f64>() / xs.len() as f64;
        xs.iter().map(|x| (x - m).powi(2)).sum::<f64>() / (xs.len() - 1) as f64
    };
    let items: f64 = (0..k).map(|j| var(&responses.iter().map(|r| r[j]).collect::<Vec<_>>())).sum();
    let total = var(&responses.iter().map(|r| r.iter().sum()).collect::<Vec<_>>());
    (total > 1e-12).then(|| k as f64 / (k - 1) as f64 * (1.0 - items / total))
}

/// The study's report: each record checked against the sealed protocol,
/// those refused with the reason, the measures per condition, each
/// condition's difference from the control on the primary measure with its
/// bootstrap interval, Cronbach's alpha of the ownership items, and the
/// Vendi score of each condition's final designs.
pub fn cohort(sealed_protocol: &Value, records: &[Value]) -> Result<Value, String> {
    let (p, digest) = open_protocol(sealed_protocol)?;
    let mut refused = Vec::new();
    let mut by_cond: BTreeMap<String, Vec<(Value, Value)>> = BTreeMap::new();
    let mut seen = BTreeSet::new();
    for (i, s) in records.iter().enumerate() {
        match open_record(s, &p, &digest).and_then(|r| measure(&r).map(|m| (r, m))) {
            Ok((r, m)) => {
                let who = r["participant"].as_str().unwrap_or_default().to_string();
                if !seen.insert(who.clone()) {
                    refused.push(json!({ "record": i, "why": format!("a second record from {who}: one record per participant") }));
                    continue;
                }
                by_cond.entry(r["condition"].as_str().unwrap_or_default().to_string()).or_default().push((r, m));
            }
            Err(e) => refused.push(json!({ "record": i, "why": e })),
        }
    }
    let primary = p.value["primary"].as_str().unwrap_or_default();
    let control = p.value["control"].as_str().unwrap_or_default();
    let a = &p.value["analysis"];
    let (resamples, seed, level) = (a["resamples"].as_u64().unwrap_or(10_000), a["seed"].as_u64().unwrap_or(0), a["level"].as_f64().unwrap_or(0.95));
    let values = |cond: &str, k: &str| -> Vec<f64> { by_cond.get(cond).into_iter().flatten().filter_map(|(_, m)| scalars(m).get(k).copied()).collect() };
    let mut conditions = Vec::new();
    for c in p.conditions() {
        let name = c["name"].as_str().unwrap_or_default();
        let ms: BTreeMap<&str, Value> = MEASURES.iter().map(|(k, _)| (*k, json!(mean(&values(name, k)).map(round)))).collect();
        let finals: Vec<Design> = by_cond.get(name).into_iter().flatten().filter_map(|(r, _)| {
            let h = History::from_json(r.get("history")?).ok()?;
            let head = h.head().ok()?;
            Some(Design { name: r["participant"].as_str()?.to_string(), source: head.source.clone(), at: Some(head.at) })
        }).collect();
        let vendi = if finals.len() >= 2 { converge::cohort(&finals, &Inputs::new(), 0).ok().map(|v| v["vendi"].clone()) } else { None };
        let mut entry = json!({ "condition": name, "participants": by_cond.get(name).map_or(0, Vec::len), "means": ms, "final_designs_vendi": vendi });
        if name != control {
            entry["primary"] = bootstrap(&values(control, primary), &values(name, primary), resamples, seed, level);
        }
        conditions.push(entry);
    }
    let answers: Vec<Vec<f64>> = by_cond.values().flatten().flat_map(|(r, _)| {
        r.get("ownership").and_then(Value::as_array).cloned().unwrap_or_default().into_iter().filter_map(|o| {
            let items: Vec<f64> = o.get("items")?.as_array()?.iter().filter_map(Value::as_f64).collect();
            (items.len() == OWNERSHIP.len()).then_some(items)
        })
    }).collect();
    Ok(json!({
        "study": p.value["study"],
        "protocol": digest,
        "hypotheses": p.value["hypotheses"],
        "primary": primary,
        "control": control,
        "analysis": { "resamples": resamples, "seed": seed, "level": level, "method": "the difference of means, each group's participants resampled with replacement" },
        "records": records.len(),
        "included": by_cond.values().map(Vec::len).sum::<usize>(),
        "refused": refused,
        "conditions": conditions,
        "ownership_alpha": cronbach(&answers).map(round),
        "ownership_answers": answers.len(),
    }))
}

/// The study as a page, an app or an agent calls it, with JSON arguments:
/// `flags` and `measures` (what a protocol may name), `probes` (the probe
/// tasks), `score_probe` (`probe`, `qasm`), `check_protocol` (`protocol`),
/// `open_protocol` (`sealed`), `assign` (`sealed`, `participant`), `measure`
/// (`record`, unsealed) and `cohort` (`sealed`, `records`). Sealing takes a
/// key, which the host holds: [`seal_protocol`] and [`seal_record`].
pub fn call(op: &str, args: &Value) -> Result<Value, String> {
    let s = |k: &str| args.get(k).ok_or_else(|| format!("{op}: give \"{k}\""));
    match op {
        "flags" => Ok(json!(FLAGS.iter().map(|(k, w)| json!({ "flag": k, "what": w })).collect::<Vec<_>>())),
        "measures" => Ok(json!(MEASURES.iter().map(|(k, w)| json!({ "measure": k, "what": w })).collect::<Vec<_>>())),
        "probes" => Ok(json!(probes().into_iter().map(|(id, what, _)| json!({ "probe": id, "task": what })).collect::<Vec<_>>())),
        "ownership_items" => Ok(json!(OWNERSHIP)),
        "score_probe" => score_probe(s("probe")?.as_str().ok_or("probe: an id")?, s("qasm")?.as_str().ok_or("qasm: the answer")?),
        "check_protocol" => Protocol::from_json(s("protocol")?).map(|_| json!({ "ok": true })),
        "open_protocol" => open_protocol(s("sealed")?).map(|(p, d)| json!({ "protocol": p.value, "digest": d })),
        "assign" => {
            let (p, d) = open_protocol(s("sealed")?)?;
            Ok(assign(&p, &d, s("participant")?.as_str().ok_or("participant: a did:key")?))
        }
        "measure" => measure(s("record")?),
        "cohort" => cohort(s("sealed")?, s("records")?.as_array().ok_or("records: a list of sealed records")?),
        _ => Err(format!("{op}: no such call on the study")),
    }
}

/// Seal a protocol or a record with `key`, for hosts that hold one:
/// `seal_protocol` (`protocol`) or `seal_record` (`record`).
pub fn call_sealing(op: &str, args: &Value, key: &SigningKey) -> Result<Value, String> {
    match op {
        "seal_protocol" => Ok(seal_protocol(&Protocol::from_json(args.get("protocol").ok_or("give \"protocol\"")?)?, key)),
        "seal_record" => seal_record(args.get("record").ok_or("give \"record\"")?, key),
        _ => call(op, args),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::engine::versions::By;

    fn key(b: u8) -> SigningKey {
        SigningKey::from_bytes(&[b; 32])
    }

    fn protocol() -> Value {
        json!({
            "study": "the assistant asks first",
            "hypotheses": ["Asking first raises unassisted skill against proposing first."],
            "primary": "unassisted",
            "conditions": [
                { "name": "proposes", "flags": { "assistant_asks": false } },
                { "name": "asks", "flags": { "assistant_asks": true } },
            ],
            "control": "proposes",
            "analysis": { "resamples": 2000, "seed": 7, "level": 0.95 },
        })
    }

    #[test]
    fn a_protocol_is_checked_sealed_and_assigns_by_key() {
        assert!(Protocol::from_json(&json!({ "study": "x" })).unwrap_err().contains("hypotheses"));
        let mut bad = protocol();
        bad["primary"] = json!("beauty");
        assert!(Protocol::from_json(&bad).unwrap_err().contains("primary measure is one of"));
        bad = protocol();
        bad["conditions"][1]["flags"] = json!({ "telepathy": true });
        assert!(Protocol::from_json(&bad).unwrap_err().contains("not a flag"));
        let p = Protocol::from_json(&protocol()).unwrap();
        let sealed = seal_protocol(&p, &key(1));
        let (back, digest) = open_protocol(&sealed).unwrap();
        assert_eq!(back, p);
        let mut tampered = sealed.clone();
        tampered["protocol"]["primary"] = json!("quality");
        assert!(open_protocol(&tampered).unwrap_err().contains("changed after it was sealed"));
        // Assignment is fixed by the protocol and the key, and spreads participants.
        let conds: BTreeSet<String> = (0..20u8).map(|b| assign(&p, &digest, &crate::quantum_receipt::did_key(&key(b + 10).verifying_key()))["condition"].as_str().unwrap().to_string()).collect();
        assert_eq!(conds.len(), 2, "{conds:?}");
        let did = crate::quantum_receipt::did_key(&key(3).verifying_key());
        assert_eq!(assign(&p, &digest, &did), assign(&p, &digest, &did));
    }

    #[test]
    fn probes_are_scored_by_the_engine() {
        let tree = "OPENQASM 3.0;\nqubit[4] q;\nh q[0];\ncx q[0], q[1];\ncx q[0], q[2];\ncx q[1], q[3];\n";
        let s = score_probe("ghz4", tree).unwrap();
        assert_eq!(s["verified"], true, "{s}");
        assert!(s["score"].as_f64().unwrap() > 1.0, "a shallower tree beats the chain: {s}");
        assert_eq!(score_probe("ghz4", "OPENQASM 3.0;\nqubit[4] q;\nh q[0];\n").unwrap()["score"], 0.0);
        assert_eq!(score_probe("swap", "OPENQASM 3.0;\nqubit[2] q;\ncx q[0], q[1];\ncx q[1], q[0];\ncx q[0], q[1];\n").unwrap()["score"], 1.0);
        let tidy = score_probe("tidy", "OPENQASM 3.0;\nqubit[2] q;\nh q[0];\ncx q[0], q[1];\nrz(0.75) q[0];\n").unwrap();
        assert_eq!(tidy["score"], 2.0, "6 gates to 3: {tidy}");
        assert!(score_probe("nonsense", "").unwrap_err().contains("no such probe"));
    }

    fn record_for(k: &SigningKey, digest: &str, p: &Protocol, unassisted: &str) -> Value {
        let did = crate::quantum_receipt::did_key(&k.verifying_key());
        let cond = assign(p, digest, &did)["condition"].clone();
        let bell = "OPENQASM 3.0;\nqubit[2] q;\nh q[0];\nx q[1];\nx q[1];\ncx q[0], q[1];\n";
        let mut h = History::start(bell, By::new(Kind::Person, None), None, 1, None).unwrap();
        h.commit("v0", "OPENQASM 3.0;\nqubit[2] q;\nh q[0];\ncx q[0], q[1];\n", By::new(Kind::Person, None), None, 2, None).unwrap();
        let r = json!({
            "protocol": digest,
            "condition": cond,
            "history": h,
            "predictions": [{ "share": 0.5, "got": 0.49, "right": true }, { "share": 0.7, "got": 0.52, "right": true }, { "share": 0.2, "got": 0.1, "right": false }],
            "choices": [{ "k": 3, "picked": 0, "best": 0 }, { "k": 2, "picked": 1, "best": 0 }],
            "probes": [{ "probe": "ghz4", "source": unassisted }],
            "ownership": [{ "items": [6, 6, 5, 7, 6] }, { "items": [5, 6, 5, 6, 5] }],
        });
        seal_record(&r, k).unwrap()
    }

    #[test]
    fn a_record_is_measured_by_the_engine() {
        let p = Protocol::from_json(&protocol()).unwrap();
        let (_, digest) = open_protocol(&seal_protocol(&p, &key(1))).unwrap();
        let chain = "OPENQASM 3.0;\nqubit[4] q;\nh q[0];\ncx q[0], q[1];\ncx q[1], q[2];\ncx q[2], q[3];\n";
        let sealed = record_for(&key(20), &digest, &p, chain);
        let r = open_record(&sealed, &p, &digest).unwrap();
        let m = measure(&r).unwrap();
        assert_eq!((m["quality"]["edits"].as_u64(), m["quality"]["improved"].as_u64()), (Some(1), Some(1)), "removing x x is an improvement the engine proves: {m}");
        assert_eq!((m["selection"]["choices"].as_u64(), m["selection"]["rate"].as_f64()), (Some(2), Some(0.5)), "{m}");
        assert_eq!(m["calibration"]["predictions"], 3);
        assert!(m["calibration"]["slope"].is_number(), "{m}");
        assert_eq!(m["unassisted"]["mean"], 1.0, "the chain is the reference: {m}");
        assert_eq!(m["ownership"]["answers"], 2);
        assert_eq!(m["authorship"], 1.0, "every gate is the person's: {m}");
        let sc = scalars(&m);
        assert!(sc.contains_key("quality") && sc.contains_key("unassisted") && sc.contains_key("ownership"), "{sc:?}");
        // A changed record, another study, another key: refused.
        let mut changed = sealed.clone();
        changed["record"]["probes"][0]["source"] = json!("OPENQASM 3.0;\nqubit[4] q;\n");
        assert!(open_record(&changed, &p, &digest).unwrap_err().contains("changed after it was sealed"));
        assert!(open_record(&sealed, &p, "00").unwrap_err().contains("another study"));
    }

    #[test]
    fn the_cohort_is_checked_compared_and_alpha_computed() {
        let p = Protocol::from_json(&protocol()).unwrap();
        let sp = seal_protocol(&p, &key(1));
        let (_, digest) = open_protocol(&sp).unwrap();
        let chain = "OPENQASM 3.0;\nqubit[4] q;\nh q[0];\ncx q[0], q[1];\ncx q[1], q[2];\ncx q[2], q[3];\n";
        let tree = "OPENQASM 3.0;\nqubit[4] q;\nh q[0];\ncx q[0], q[1];\ncx q[0], q[2];\ncx q[1], q[3];\n";
        let mut records = Vec::new();
        for b in 30..46u8 {
            let k = key(b);
            let did = crate::quantum_receipt::did_key(&k.verifying_key());
            // Those who are asked first answer the probe with the tree.
            let asks = assign(&p, &digest, &did)["condition"] == "asks";
            records.push(record_for(&k, &digest, &p, if asks { tree } else { chain }));
        }
        records.push(records[0].clone());
        let mut wrong = records[1].clone();
        let flip = if wrong["record"]["condition"] == "asks" { "proposes" } else { "asks" };
        wrong["record"]["condition"] = json!(flip);
        records.push(wrong);
        let r = cohort(&sp, &records).unwrap();
        assert_eq!((r["records"].as_u64(), r["included"].as_u64()), (Some(18), Some(16)), "{r}");
        let why: Vec<&str> = r["refused"].as_array().unwrap().iter().map(|x| x["why"].as_str().unwrap()).collect();
        assert!(why.iter().any(|w| w.contains("second record")) && why.iter().any(|w| w.contains("changed after it was sealed") || w.contains("is assigned")), "{why:?}");
        let asks = r["conditions"].as_array().unwrap().iter().find(|c| c["condition"] == "asks").unwrap();
        let d = &asks["primary"];
        assert!(d["difference"].as_f64().unwrap() > 0.0 && d["excludes_zero"] == true, "the tree's score beats the chain's: {r}");
        assert!(r["ownership_alpha"].is_number(), "{r}");
        // The same inputs give the same report.
        assert_eq!(cohort(&sp, &records).unwrap(), r);
    }
}