55208351c3
- A named method that needs standard gates reads them, so a program they cannot hold is refused as the reader refuses it: on its line, with Q208 or Q210 and no J011. Both refusals past 16 controls say how a job holds the gate, and no longer say the bit-exact method does so by default. - A verdict with no intent reads the program as it ran, so a gate past 16 controls runs over MCP, as the docs say; an intent is checked in standard gates, and a job with one is refused there, on its line. - lint's dynamic_steps counts standard steps, beside gates. - options.controls is read: another value, or "whole" on a job that runs standard gates, is refused (J003), and the schema, methods list, API description, README and both MCP servers name it. - accuracy.global_phase is stated against method statevector's state, not the language's, whose U and gphase differ from it; below_table and moved_angles say they count phase gates, at 1e-14 of a turn. - operations is an engine member. - Gates held whole may take 2,000,000 qubits in all (Q210), which holds nothing the standard form holds. - A metered plan says `metered`, its work what the meter allows, and such a job does not set the method's measured rate. - The refusal, service and QEC pages and the corpus say what changed; the tests are named in CI; the conformance set holds the new text forms; the service snapshot and the agent docs are regenerated. Co-Authored-By: Claude Opus 5.5 <[email protected]>
205 lines
11 KiB
Python
205 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""The README's results section, from the committed results files.
|
|
|
|
report.py rewrite the section between the README's results markers
|
|
report.py --check exit 1 when the section is not what the results files give
|
|
|
|
Standard library only. The tables are written from results/*.json and nothing else, so the
|
|
page says what the committed runs found, and CI checks that it still does.
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.dirname(HERE)
|
|
README = os.path.join(ROOT, "README.md")
|
|
BEGIN = "<!-- results: written by tools/report.py from results/*.json; do not edit by hand -->"
|
|
END = "<!-- end of results -->"
|
|
|
|
ENGINE_ORDER = ["wai.exact", "wai.statevector", "wai.stabilizer", "wai.auto", "wai.mps", "wai.tensor_network", "wai.density_matrix", "wai.qec", "aer.statevector", "aer.qasm_import", "aer.stabilizer", "aer.mps", "aer.density_matrix", "stim.tableau", "quimb.dense", "quimb.tensor_network", "quimb.mps", "ddsim.statevector", "qsim.statevector"]
|
|
|
|
|
|
def family_of(entry, families):
|
|
return families.get(entry, "qec" if entry.startswith(("surface", "unrotated", "repetition", "color", "qec_")) else "other")
|
|
|
|
|
|
def fmt(x):
|
|
if x is None:
|
|
return ""
|
|
if isinstance(x, float):
|
|
return "%.2g" % x
|
|
return str(x)
|
|
|
|
|
|
def fmt_seconds(x):
|
|
return "" if x is None else "%.3g" % x
|
|
|
|
|
|
def fmt_mib(x):
|
|
return "" if x is None else "%.0f" % x
|
|
|
|
|
|
def host_line(r):
|
|
h = r["host"]
|
|
mem = h.get("memory_bytes")
|
|
return "%s, %s, %s logical CPUs, %s GiB, %s" % (h.get("cpu", "?"), h.get("os", h.get("system")), h.get("logical_cpus"), round(mem / 2**30) if mem else "?", h.get("machine"))
|
|
|
|
|
|
def results_section():
|
|
manifest = json.load(open(os.path.join(ROOT, "corpus", "manifest.json")))
|
|
families = {r["id"]: r["family"] for r in manifest["circuits"]}
|
|
out = []
|
|
runs = sorted(glob.glob(os.path.join(ROOT, "results", "conformance-*.json")))
|
|
for path in runs:
|
|
r = json.load(open(path))
|
|
name = os.path.basename(path)
|
|
s = r["summary"]
|
|
out.append("### `%s`" % name)
|
|
out.append("")
|
|
out.append("Host: %s. Tier %s, %s s. Checks: %s." % (host_line(r), r["tier"], r["seconds"], ", ".join("%s %d" % kv for kv in sorted(s.items()))))
|
|
out.append("")
|
|
# Per family and engine: checks passed of those made, and the largest measured/tolerance.
|
|
table = {}
|
|
for c in r["checks"]:
|
|
eng = c["engine"]
|
|
if "~" in eng or eng in ("exact", "corpus.py"):
|
|
eng = "pairs and references" if "~" in eng else "references"
|
|
fam = family_of(c["entry"], families)
|
|
cell = table.setdefault((fam, eng), {"pass": 0, "fail": 0, "not run": 0, "refused": 0, "info": 0, "known": 0, "ratio": 0.0})
|
|
cell[c["status"]] = cell.get(c["status"], 0) + 1
|
|
m, t = c.get("measured"), c.get("tolerance")
|
|
# A measured magnitude under a bound: both are written as floats. (Counts and
|
|
# ranks held to equality are integers, and are not ratios.)
|
|
if isinstance(m, float) and isinstance(t, float) and t > 0 and c["status"] == "pass":
|
|
cell["ratio"] = max(cell["ratio"], m / t)
|
|
fams = sorted({f for f, _ in table})
|
|
engs = [e for e in ENGINE_ORDER + ["pairs and references", "references"] if any((f, e) in table for f in fams)]
|
|
out.append("| family | " + " | ".join("`%s`" % e for e in engs) + " |")
|
|
out.append("|---|" + "---|" * len(engs))
|
|
for f in fams:
|
|
cells = []
|
|
for e in engs:
|
|
c = table.get((f, e))
|
|
if not c:
|
|
cells.append("")
|
|
continue
|
|
made = c["pass"] + c["fail"]
|
|
txt = "%d/%d" % (c["pass"], made) if made else ""
|
|
if c["ratio"]:
|
|
txt += " (%s)" % fmt(c["ratio"])
|
|
if c["not run"]:
|
|
txt += " · %d not run" % c["not run"]
|
|
if c["refused"]:
|
|
txt += " · %d refused as listed" % c["refused"]
|
|
if c.get("known"):
|
|
txt += " · %d known" % c["known"]
|
|
cells.append(txt.strip(" ·"))
|
|
out.append("| %s | %s |" % (f, " | ".join(cells)))
|
|
out.append("")
|
|
fails = [c for c in r["checks"] if c["status"] == "fail"]
|
|
if fails:
|
|
out.append("Failures:")
|
|
out.append("")
|
|
for c in fails:
|
|
out.append("- `%s`, `%s`: %s — measured %s, tolerance %s%s" % (c["entry"], c["engine"], c["quantity"], fmt(c["measured"]), fmt(c["tolerance"]), (". " + c["note"]) if c.get("note") else ""))
|
|
out.append("")
|
|
known = [c for c in r["checks"] if c["status"] == "known"]
|
|
if known:
|
|
out.append("Known disagreements of other engines with the exact reference (`corpus/known-disagreements.json`):")
|
|
out.append("")
|
|
for c in known:
|
|
if c["quantity"].startswith("reproducer"):
|
|
# A case of a committed reproducer, rerun: its claim and what it saw.
|
|
out.append("- `%s`, `%s`, %s — saw %s" % (c["entry"], c["engine"], c["quantity"], str(c["measured"])[:160].replace("\n", " ")))
|
|
else:
|
|
out.append("- `%s`, `%s`: %s" % (c["entry"], c["engine"], (c.get("note") or "").replace("\n", " ")))
|
|
out.append("")
|
|
refused = [c for c in r["checks"] if c["status"] == "refused"]
|
|
if refused:
|
|
out.append("<details><summary>Refusals by the engine under test, each listed in <code>corpus/expected-refusals.json</code> with its words (%d)</summary>" % len(refused))
|
|
out.append("")
|
|
for c in refused:
|
|
out.append("- `%s`, `%s`: %s" % (c["entry"], c["engine"], str(c["measured"])[:300].replace("\n", " ")))
|
|
out.append("")
|
|
out.append("</details>")
|
|
out.append("")
|
|
limits = [c for c in r["checks"] if c["status"] == "not run"]
|
|
if limits:
|
|
out.append("<details><summary>Checks other engines did not make, and why (%d)</summary>" % len(limits))
|
|
out.append("")
|
|
for c in limits:
|
|
out.append("- `%s`, `%s`: %s" % (c["entry"], c["engine"], str(c.get("note") or "")[:300].replace("\n", " ")))
|
|
out.append("")
|
|
out.append("</details>")
|
|
out.append("")
|
|
infos = [c for c in r["checks"] if c["status"] == "info" and not c["quantity"].startswith("distinct outcomes")]
|
|
if infos:
|
|
out.append("<details><summary>Recorded and not judged (%d)</summary>" % len(infos))
|
|
out.append("")
|
|
for c in infos:
|
|
out.append("- `%s`, `%s`: %s — measured %s%s" % (c["entry"], c["engine"], c["quantity"], fmt(c["measured"]), (". " + c["note"].replace("\n", " ")[:300]) if c.get("note") else ""))
|
|
out.append("")
|
|
out.append("</details>")
|
|
out.append("")
|
|
out.append("Versions: " + ", ".join("%s %s" % kv for kv in sorted(r["versions"].items())) + ".")
|
|
out.append("")
|
|
perfs = sorted(glob.glob(os.path.join(ROOT, "results", "performance-*.json")))
|
|
for path in perfs:
|
|
p = json.load(open(path))
|
|
out.append("### `%s`" % os.path.basename(path))
|
|
out.append("")
|
|
out.append("Host: %s. Each figure is the engine's own call, the median of %d runs (the minimum in brackets). Memory is the peak resident set of the process that ran the engine, and the baseline the same given a two-qubit job. Ops are the operations the engine ran, where it reports them." % (host_line(p), p["repeat"]))
|
|
out.append("")
|
|
for k, v in sorted(p.get("timers", {}).items()):
|
|
out.append("- `%s.*` seconds: %s." % (k, v))
|
|
if p.get("timers"):
|
|
out.append("")
|
|
out.append("| case | engine | threads | ops | seconds, median (min) | peak MiB (baseline) | load average before → after |")
|
|
out.append("|---|---|---|---|---|---|---|")
|
|
for row in p["rows"]:
|
|
if row.get("not_run"):
|
|
out.append("| %s | `%s` | %s | | not run: %s | | |" % (row["case"], row["engine"], row["threads"], row["not_run"][:120]))
|
|
continue
|
|
sec = "%s (%s)" % (fmt_seconds(row["seconds_median"]), fmt_seconds(row["seconds_min"]))
|
|
mem = "%s (%s)" % (fmt_mib(row["peak_rss_mib"]), fmt_mib(row.get("baseline_rss_mib")))
|
|
out.append("| %s | `%s` | %s | %s | %s | %s | %s → %s |" % (row["case"], row["engine"], row["threads"], row.get("ops") or "", sec, mem, row["load_before"][0], row["load_after"][0]))
|
|
out.append("")
|
|
for path in sorted(glob.glob(os.path.join(ROOT, "results", "before-after-*.json"))):
|
|
p = json.load(open(path))
|
|
out.append("### `%s`" % os.path.basename(path))
|
|
out.append("")
|
|
out.append("Host: %s. The same harness and circuits, with the engine under test built before and after a gate under many controls stood whole on its state vector, one build straight after the other: each figure is the engine's own call, the median of %d runs (the minimum in brackets), and ops are the operations it ran." % (host_line(p), p["repeat"]))
|
|
out.append("")
|
|
out.append("`wai.exact_whole`'s before rows ran a build with no such option, so they are standard gates, as their ops show. The last column is the one-minute load as each build's row began.")
|
|
out.append("")
|
|
out.append("| case | engine | threads | before: ops | before: seconds, median (min) | after: ops | after: seconds, median (min) | load average, before build → after build |")
|
|
out.append("|---|---|---|---|---|---|---|---|")
|
|
for row in p["rows"]:
|
|
cell = lambda r: ("not run: %s" % r["not_run"][:90]) if r.get("not_run") else "%s (%s)" % (fmt_seconds(r["seconds_median"]), fmt_seconds(r["seconds_min"]))
|
|
out.append("| %s | `%s` | %s | %s | %s | %s | %s | %s → %s |" % (row["case"], row["engine"], row["threads"], row["before"].get("ops") or "", cell(row["before"]), row["after"].get("ops") or "", cell(row["after"]), row["before"]["load_before"][0], row["after"]["load_before"][0]))
|
|
out.append("")
|
|
return "\n".join(out).rstrip() + "\n"
|
|
|
|
|
|
def main():
|
|
text = open(README).read()
|
|
if BEGIN not in text or END not in text:
|
|
print("report.py: the README has no results markers", file=sys.stderr)
|
|
sys.exit(2)
|
|
head, rest = text.split(BEGIN, 1)
|
|
_, tail = rest.split(END, 1)
|
|
new = head + BEGIN + "\n\n" + results_section() + "\n" + END + tail
|
|
if "--check" in sys.argv:
|
|
if new != text:
|
|
print("report.py: the README's results section is not what results/*.json give; run tools/report.py", file=sys.stderr)
|
|
sys.exit(1)
|
|
print("report.py: the README's results section matches results/*.json")
|
|
return
|
|
with open(README, "w") as f:
|
|
f.write(new)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|