#!/usr/bin/env python3
"""The README's results section, from the committed results files.
report.py rewrite the section between the README's results markers
report.py --check exit 1 when the section is not what the results files give
Standard library only. The tables are written from results/*.json and nothing else, so the
page says what the committed runs found, and CI checks that it still does.
"""
import glob
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
README = os.path.join(ROOT, "README.md")
BEGIN = ""
END = ""
ENGINE_ORDER = ["wai.exact", "wai.statevector", "wai.stabilizer", "wai.auto", "wai.mps", "wai.tensor_network", "wai.density_matrix", "wai.qec", "wai.pauli", "wai.stabrank", "wai.cutrank", "wai.trajectories", "wai.mitigate", "wai.pulse", "wai.matching", "wai.union_find", "wai.statevector_single", "wai.cutrank_single", "aer.statevector", "aer.qasm_import", "aer.stabilizer", "aer.mps", "aer.density_matrix", "stim.tableau", "quimb.dense", "quimb.tensor_network", "quimb.mps", "ddsim.statevector", "qsim.statevector"]
def family_of(entry, families):
return families.get(entry, "qec" if entry.startswith(("surface", "unrotated", "repetition", "color", "qec_")) else "other")
def fmt(x):
if x is None:
return ""
if isinstance(x, float):
return "%.2g" % x
return str(x)
def fmt_seconds(x):
return "" if x is None else "%.3g" % x
def fmt_mib(x):
return "" if x is None else "%.0f" % x
def host_line(r):
h = r["host"]
mem = h.get("memory_bytes")
return "%s, %s, %s logical CPUs, %s GiB, %s" % (h.get("cpu", "?"), h.get("os", h.get("system")), h.get("logical_cpus"), round(mem / 2**30) if mem else "?", h.get("machine"))
def results_section():
manifest = json.load(open(os.path.join(ROOT, "corpus", "manifest.json")))
families = {r["id"]: r["family"] for r in manifest["circuits"]}
out = []
runs = sorted(glob.glob(os.path.join(ROOT, "results", "conformance-*.json")))
for path in runs:
r = json.load(open(path))
name = os.path.basename(path)
s = r["summary"]
out.append("### `%s`" % name)
out.append("")
out.append("Host: %s. Tier %s, %s s. Checks: %s." % (host_line(r), r["tier"], r["seconds"], ", ".join("%s %d" % kv for kv in sorted(s.items()))))
out.append("")
# Per family and engine: checks passed of those made, and the largest measured/tolerance.
table = {}
for c in r["checks"]:
eng = c["engine"]
if "~" in eng or eng in ("exact", "corpus.py"):
eng = "pairs and references" if "~" in eng else "references"
fam = family_of(c["entry"], families)
cell = table.setdefault((fam, eng), {"pass": 0, "fail": 0, "not run": 0, "refused": 0, "info": 0, "known": 0, "ratio": 0.0})
cell[c["status"]] = cell.get(c["status"], 0) + 1
m, t = c.get("measured"), c.get("tolerance")
# A measured magnitude under a bound: both are written as floats. (Counts and
# ranks held to equality are integers, and are not ratios.)
if isinstance(m, float) and isinstance(t, float) and t > 0 and c["status"] == "pass":
cell["ratio"] = max(cell["ratio"], m / t)
fams = sorted({f for f, _ in table})
engs = [e for e in ENGINE_ORDER + ["pairs and references", "references"] if any((f, e) in table for f in fams)]
out.append("| family | " + " | ".join("`%s`" % e for e in engs) + " |")
out.append("|---|" + "---|" * len(engs))
for f in fams:
cells = []
for e in engs:
c = table.get((f, e))
if not c:
cells.append("")
continue
made = c["pass"] + c["fail"]
txt = "%d/%d" % (c["pass"], made) if made else ""
if c["ratio"]:
txt += " (%s)" % fmt(c["ratio"])
if c["not run"]:
txt += " · %d not run" % c["not run"]
if c["refused"]:
txt += " · %d refused as listed" % c["refused"]
if c.get("known"):
txt += " · %d known" % c["known"]
cells.append(txt.strip(" ·"))
out.append("| %s | %s |" % (f, " | ".join(cells)))
out.append("")
fails = [c for c in r["checks"] if c["status"] == "fail"]
if fails:
out.append("Failures:")
out.append("")
for c in fails:
out.append("- `%s`, `%s`: %s — measured %s, tolerance %s%s" % (c["entry"], c["engine"], c["quantity"], fmt(c["measured"]), fmt(c["tolerance"]), (". " + c["note"]) if c.get("note") else ""))
out.append("")
known = [c for c in r["checks"] if c["status"] == "known"]
if known:
out.append("Known disagreements of other engines with the exact reference (`corpus/known-disagreements.json`):")
out.append("")
for c in known:
if c["quantity"].startswith("reproducer"):
# A case of a committed reproducer, rerun: its claim and what it saw.
out.append("- `%s`, `%s`, %s — saw %s" % (c["entry"], c["engine"], c["quantity"], str(c["measured"])[:160].replace("\n", " ")))
else:
out.append("- `%s`, `%s`: %s" % (c["entry"], c["engine"], (c.get("note") or "").replace("\n", " ")))
out.append("")
refused = [c for c in r["checks"] if c["status"] == "refused"]
if refused:
out.append("Refusals by the engine under test, each listed in corpus/expected-refusals.json with its words (%d)
" % len(refused))
out.append("")
for c in refused:
out.append("- `%s`, `%s`: %s" % (c["entry"], c["engine"], str(c["measured"])[:300].replace("\n", " ")))
out.append("")
out.append("Checks other engines did not make, and why (%d)
" % len(limits))
out.append("")
for c in limits:
out.append("- `%s`, `%s`: %s" % (c["entry"], c["engine"], str(c.get("note") or "")[:300].replace("\n", " ")))
out.append("")
out.append("Recorded and not judged (%d)
" % len(infos))
out.append("")
for c in infos:
out.append("- `%s`, `%s`: %s — measured %s%s" % (c["entry"], c["engine"], c["quantity"], fmt(c["measured"]), (". " + c["note"].replace("\n", " ")[:300]) if c.get("note") else ""))
out.append("")
out.append("