55208351c3
- A named method that needs standard gates reads them, so a program they cannot hold is refused as the reader refuses it: on its line, with Q208 or Q210 and no J011. Both refusals past 16 controls say how a job holds the gate, and no longer say the bit-exact method does so by default. - A verdict with no intent reads the program as it ran, so a gate past 16 controls runs over MCP, as the docs say; an intent is checked in standard gates, and a job with one is refused there, on its line. - lint's dynamic_steps counts standard steps, beside gates. - options.controls is read: another value, or "whole" on a job that runs standard gates, is refused (J003), and the schema, methods list, API description, README and both MCP servers name it. - accuracy.global_phase is stated against method statevector's state, not the language's, whose U and gphase differ from it; below_table and moved_angles say they count phase gates, at 1e-14 of a turn. - operations is an engine member. - Gates held whole may take 2,000,000 qubits in all (Q210), which holds nothing the standard form holds. - A metered plan says `metered`, its work what the meter allows, and such a job does not set the method's measured rate. - The refusal, service and QEC pages and the corpus say what changed; the tests are named in CI; the conformance set holds the new text forms; the service snapshot and the agent docs are regenerated. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1309 lines
66 KiB
Python
1309 lines
66 KiB
Python
#!/usr/bin/env python3
|
||
"""Run every engine on the corpus, hold each answer to its reference within its tolerance, and
|
||
say what passed, what failed and by how much.
|
||
|
||
compare.py --tier ci|full [--out RESULTS.json] [--jobs N] [--inject FAULT] [--only ID,...]
|
||
|
||
Exit status 0 when every check passes, 1 when one fails, 2 when the run itself could not be
|
||
made. `--inject` corrupts one engine's answers after it gives them, or makes the engine under
|
||
test refuse, crash, hang or accept what it must not, the way a wrong engine would, so that a run
|
||
can be seen to fail; FAULTS lists what it can do.
|
||
|
||
Every tolerance is derived, never fitted, and tolerance.py says from what. A check another engine
|
||
cannot make (an engine without the gate, a method that does not take the circuit) is recorded as
|
||
`not run`, with the engine's own words, and is not a pass. The engine under test is held to more:
|
||
a crash, a time-out, or a refusal that corpus/expected-refusals.json does not list in those words
|
||
fails the run, and so does an answer to a circuit that file says it must refuse.
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import itertools
|
||
import json
|
||
import math
|
||
import os
|
||
import platform
|
||
import shutil
|
||
import signal
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from fractions import Fraction
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
sys.path.insert(0, HERE)
|
||
import corpus # noqa: E402
|
||
import exact # noqa: E402
|
||
import tolerance as tol # noqa: E402
|
||
|
||
ROOT = os.path.dirname(HERE)
|
||
WAI = os.path.dirname(ROOT)
|
||
|
||
STATE_ENGINES = ["wai.exact", "wai.statevector", "aer.statevector", "aer.qasm_import", "quimb.dense", "quimb.tensor_network", "ddsim.statevector", "qsim.statevector"]
|
||
COUNT_ENGINES = ["wai.auto", "wai.statevector", "aer.statevector", "ddsim.statevector", "qsim.statevector"]
|
||
CLIFFORD = {"id", "x", "y", "z", "h", "s", "sdg", "sx", "sxdg", "cx", "cy", "cz", "swap"}
|
||
# Every sampled test of a run (counts, error-correction rates, stabilizer outcomes) shares this:
|
||
# a correct engine fails a run on sampling with probability at most this, a third to each kind.
|
||
FAMILY_WISE_ALPHA = 1e-6
|
||
# The closed-form checks of the integer reference, in units of 2^-256: far above its own rounding
|
||
# (a few units a gate), far below anything an engine is held to.
|
||
REFERENCE_UNITS = 2**40
|
||
# mqt.ddsim's angle tolerance (its parameter_tolerance setting): an angle below it may be read as 0.
|
||
DDSIM_PARAMETER_TOLERANCE = 1e-13
|
||
|
||
FAULTS = {
|
||
"amplitude": "aer.statevector: one amplitude of every state moved by 1e-9",
|
||
"bit_order": "wai.statevector: every state read with its qubits in reverse order",
|
||
"counts": "stim.tableau: 5% of the shots of the most frequent outcome moved to the least frequent outcome that can occur",
|
||
"counts_spread": "aer.statevector: every outcome's count moved by a tenth of itself, up and down in turn, the total kept (an L1 distance of about 0.1)",
|
||
"dem": "wai.qec: the most likely mechanism's probability raised by one part in 10^6",
|
||
"qec_rate": "wai.qec: the sampler's most frequent detector count raised by a quarter",
|
||
"stabilizer": "stim.tableau: the sign of the last stabilizer flipped",
|
||
"fixed_point": "wai.exact: the lowest bit of one fixed-point amplitude flipped",
|
||
"drop_gate": "wai.exact: the op-log's last phase dropped, and the integers and hash made again from what is left",
|
||
"drop_phase": "wai.statevector: the state of the circuit with its small-angle gate left out",
|
||
"refuse": "wai.mps: the engine refuses every job",
|
||
"crash": "wai.auto: the engine exits with status 101 and no answer",
|
||
"hang": "wai.qec: the engine's sampler never answers",
|
||
"accept": "wai.exact: every angle moved to the nearest m/2^32 of a turn before the engine reads it",
|
||
"qec_words": "wai.qec: the engine refuses each model with a line number, in words no list holds",
|
||
"qec_crash": "wai.qec: the engine crashes on each model",
|
||
"exact_words": "wai.exact: the engine refuses each state, in words other than its list's",
|
||
"qec_short": "wai.qec: the sampler's answer lists no observable",
|
||
"exact_turn": "wai.exact: every state turned by a global phase of 0.5 rad",
|
||
"phase_flip": "wai.statevector: every state turned by a half turn, as a phase applied with the wrong sign, or not at all",
|
||
"mps_phase": "aer.mps: every amplitude turned by 0.5 rad",
|
||
"counts_collapse": "stim.tableau: every shot given to one outcome",
|
||
}
|
||
# The faults the engine's adapter carries out (engines.py), rather than this file.
|
||
DRIVER_FAULTS = {"refuse": "wai.mps", "crash": "wai.auto", "hang": "wai.qec", "accept": "wai.exact", "qec_words": "wai.qec", "qec_crash": "wai.qec", "exact_words": "wai.exact"}
|
||
|
||
|
||
def is_clifford(e):
|
||
for s in e["steps"]:
|
||
for x in s.get("then", [s]):
|
||
if "g" in x and x["g"] not in CLIFFORD:
|
||
return False
|
||
return True
|
||
|
||
|
||
def has_dynamic(e):
|
||
k = len(e["steps"])
|
||
while k and "measure" in e["steps"][k - 1]:
|
||
k -= 1
|
||
return any("g" not in s for s in e["steps"][:k]) or k < len(e["steps"]) and e["expect"] != "distribution"
|
||
|
||
|
||
class Runner:
|
||
def __init__(self, work, python, timeout, inject):
|
||
self.work = work
|
||
self.python = python
|
||
self.timeout = timeout
|
||
self.inject = inject
|
||
self.counter = itertools.count(1)
|
||
self.lock = threading.Lock()
|
||
|
||
def run(self, engine, job):
|
||
"""One engine on one job in a process group of its own: its answer and wall time. On a
|
||
time-out the whole group is killed, the engine's driver with its adapter."""
|
||
with self.lock:
|
||
k = next(self.counter)
|
||
tag = "%s-%s-%d" % (job.get("id", "job"), engine, k)
|
||
jp = os.path.join(self.work, tag + ".job.json")
|
||
ap = os.path.join(self.work, tag + ".answer.json")
|
||
if job["kind"] == "state" and "state_out" not in job:
|
||
job = dict(job, state_out=os.path.join(self.work, tag + ".state"))
|
||
if self.inject in DRIVER_FAULTS and DRIVER_FAULTS[self.inject] == engine:
|
||
job = dict(job, inject=self.inject)
|
||
with open(jp, "w") as f:
|
||
json.dump(job, f)
|
||
t0 = time.perf_counter()
|
||
p = subprocess.Popen([self.python, "-I", os.path.join(HERE, "engines.py"), engine, jp, ap], stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True)
|
||
try:
|
||
_, err = p.communicate(timeout=self.timeout)
|
||
except subprocess.TimeoutExpired:
|
||
try:
|
||
os.killpg(p.pid, signal.SIGKILL)
|
||
except ProcessLookupError:
|
||
pass
|
||
p.communicate()
|
||
return {"ok": False, "timeout": self.timeout}
|
||
wall = time.perf_counter() - t0
|
||
try:
|
||
ans = json.load(open(ap))
|
||
except Exception: # noqa: BLE001
|
||
return {"ok": False, "error": "no answer; exit %d: %s" % (p.returncode, err.decode(errors="replace")[-1500:])}
|
||
ans["wall_seconds"] = wall
|
||
ans["job"] = job
|
||
return self.corrupt(engine, ans)
|
||
|
||
def corrupt(self, engine, ans):
|
||
"""The injected fault, applied as the engine's own answer."""
|
||
f = self.inject
|
||
if not f or not ans.get("ok"):
|
||
return ans
|
||
import numpy as np
|
||
|
||
job = ans["job"]
|
||
if f == "amplitude" and engine == "aer.statevector" and job["kind"] == "state":
|
||
v = np.fromfile(job["state_out"], dtype=np.complex128)
|
||
v[len(v) // 2] += 1e-9
|
||
v.tofile(job["state_out"])
|
||
elif f == "bit_order" and engine == "wai.statevector" and job["kind"] == "state" and job["n"] > 1:
|
||
import engines
|
||
|
||
v = np.fromfile(job["state_out"], dtype=np.complex128)
|
||
engines.reverse_qubits(v, job["n"]).tofile(job["state_out"])
|
||
elif f == "counts" and engine == "stim.tableau" and job["kind"] == "counts":
|
||
c = ans["counts"]
|
||
possible = [k for k, p in exact.distribution(job["n"], job["bits"], job["steps"]).items() if p > 0]
|
||
top = max(c, key=c.get)
|
||
dest = min((k for k in possible if k != top), key=lambda k: c.get(k, 0), default=None)
|
||
if dest is not None:
|
||
moved = c[top] // 20
|
||
c[top] -= moved
|
||
c[dest] = c.get(dest, 0) + moved
|
||
elif f == "counts_spread" and engine == "aer.statevector" and job["kind"] == "counts":
|
||
c = ans["counts"]
|
||
keys = sorted(c)
|
||
total = sum(c.values())
|
||
for i, k in enumerate(keys):
|
||
c[k] += (c[k] // 10) * (1 if i % 2 == 0 else -1)
|
||
c[keys[0]] += total - sum(c.values())
|
||
elif f == "dem" and engine == "wai.qec" and job["kind"] == "qec_model" and ans["mechanisms"]:
|
||
m = max(ans["mechanisms"], key=lambda x: x[0])
|
||
m[0] *= 1 + 1e-6
|
||
elif f == "qec_rate" and engine == "wai.qec" and job["kind"] == "qec_sample" and ans.get("detectors"):
|
||
d = ans["detectors"]
|
||
i = max(range(len(d)), key=lambda j: d[j])
|
||
d[i] += d[i] // 4
|
||
elif f == "stabilizer" and engine == "stim.tableau" and job["kind"] == "stabilizers":
|
||
last = ans["stabilizers"][-1]
|
||
ans["stabilizers"][-1] = ("-" if last[0] == "+" else "+") + last[1:]
|
||
elif f == "fixed_point" and engine == "wai.exact" and job["kind"] == "state":
|
||
fx = ans["settings"]["fixed"]
|
||
a = np.load(fx)
|
||
a[0, 0] ^= 1
|
||
np.save(fx, a)
|
||
elif f == "drop_gate" and engine == "wai.exact" and job["kind"] == "state":
|
||
ops = ans["ops"]
|
||
last = max((i for i, op in enumerate(ops) if op[0] == 9), default=None)
|
||
if last is not None:
|
||
del ops[last]
|
||
ref = referee()
|
||
amps = ref.simulate(job["n"], [ref.gate(op, t, tuple(c), p) for op, c, t, p in ops], ref.derive_v2()[0])
|
||
ints = np.array([[int(a), int(b)] for a, b in amps], dtype=np.int64)
|
||
np.save(ans["settings"]["fixed"], ints)
|
||
digest = ref.statevector_hash(job["n"], amps)
|
||
ans["state_hash"] = digest.hex() if isinstance(digest, (bytes, bytearray)) else digest
|
||
((ints[:, 0] / float(1 << 30)) + 1j * (ints[:, 1] / float(1 << 30))).tofile(job["state_out"])
|
||
elif f == "qec_short" and engine == "wai.qec" and job["kind"] == "qec_sample":
|
||
ans["observables"] = []
|
||
elif f == "exact_turn" and engine == "wai.exact" and job["kind"] == "state":
|
||
fx = ans["settings"]["fixed"]
|
||
a = np.load(fx)
|
||
z = (a[:, 0] + 1j * a[:, 1]) * complex(math.cos(0.5), math.sin(0.5))
|
||
a[:, 0], a[:, 1] = np.round(z.real).astype(np.int64), np.round(z.imag).astype(np.int64)
|
||
np.save(fx, a)
|
||
((a[:, 0] / float(1 << 30)) + 1j * (a[:, 1] / float(1 << 30))).tofile(job["state_out"])
|
||
elif f == "phase_flip" and engine == "wai.statevector" and job["kind"] == "state":
|
||
v = np.fromfile(job["state_out"], dtype=np.complex128)
|
||
(-v).tofile(job["state_out"])
|
||
elif f == "mps_phase" and engine == "aer.mps" and job["kind"] == "amplitudes":
|
||
t = complex(math.cos(0.5), math.sin(0.5))
|
||
ans["amplitudes"] = {b: [(complex(*a) * t).real, (complex(*a) * t).imag] for b, a in ans["amplitudes"].items()}
|
||
elif f == "counts_collapse" and engine == "stim.tableau" and job["kind"] == "counts":
|
||
c = ans["counts"]
|
||
ans["counts"] = {sorted(c)[0]: sum(c.values())}
|
||
elif f == "drop_phase" and engine == "wai.statevector" and job["kind"] == "state" and job.get("probe") is not None:
|
||
steps = [s for i, s in enumerate(job["steps"]) if i != job["probe"]]
|
||
w = exact.run_unitary(job["n"], steps)
|
||
np.array([complex(float(Fraction(r, exact.ONE)), float(Fraction(i, exact.ONE))) for r, i in zip(w.re, w.im)]).tofile(job["state_out"])
|
||
return ans
|
||
|
||
|
||
_REFEREE = None
|
||
|
||
|
||
def referee():
|
||
"""The conformance corpus's independent integer reader of the bit-exact method (section 3.0)."""
|
||
global _REFEREE
|
||
if _REFEREE is None:
|
||
import importlib.util
|
||
|
||
spec = importlib.util.spec_from_file_location("referee", os.path.join(WAI, "quantum-conformance", "tools", "referee.py"))
|
||
_REFEREE = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(_REFEREE)
|
||
return _REFEREE
|
||
|
||
|
||
class Report:
|
||
def __init__(self):
|
||
self.checks = []
|
||
|
||
def add(self, entry, engine, quantity, measured, tolerance, ok, note=None, status=None):
|
||
c = {"entry": entry, "engine": engine, "quantity": quantity, "measured": measured, "tolerance": tolerance, "status": status or ("pass" if ok else "fail")}
|
||
if note:
|
||
c["note"] = note
|
||
self.checks.append(c)
|
||
return c
|
||
|
||
def not_run(self, entry, engine, quantity, why):
|
||
return self.add(entry, engine, quantity, None, None, True, note=why, status="not run")
|
||
|
||
|
||
# ------------------------------------------------------------------ the engine under test, held to its answers
|
||
|
||
|
||
class Expected:
|
||
"""corpus/expected-refusals.json: the refusals by the engine under test a run accepts."""
|
||
|
||
def __init__(self):
|
||
self.rows = json.load(open(os.path.join(ROOT, "corpus", "expected-refusals.json")))["refusals"]
|
||
self.by = {(r["entry"], r["engine"]): r for r in self.rows}
|
||
self.answered = set()
|
||
|
||
def get(self, entry, engine):
|
||
return self.by.get((entry, engine))
|
||
|
||
|
||
EXPECTED = None
|
||
|
||
|
||
def under_test(engine):
|
||
return engine.startswith("wai.")
|
||
|
||
|
||
def words(ans):
|
||
return ans.get("refused") or ans.get("unsupported") or ans.get("error") or ("did not finish within %s s" % ans.get("timeout"))
|
||
|
||
|
||
def no_answer(rep, entry, eng, ans, quantity, fail_on_error=False):
|
||
"""An engine gave no answer. From the engine under test, a refusal that expected-refusals.json
|
||
lists for this entry, in its listed words, is recorded as `refused`; anything else (a crash, a
|
||
time-out, a refusal not listed, or in other words) fails the run. From another engine it is
|
||
`not run`, with its words, or a failure where it crashed on a check it must make."""
|
||
why = str(words(ans))
|
||
if under_test(eng):
|
||
x = EXPECTED.get(entry, eng)
|
||
if x and ans.get("refused") and x["text"] in ans["refused"]:
|
||
return rep.add(entry, eng, "refuses, as listed: " + x["reason"], ans["refused"], x["text"], True, status="refused")
|
||
kind = "it crashed" if ans.get("error") else "it did not finish" if ans.get("timeout") else "a refusal no list holds" if not x else "a refusal in other words than its list's"
|
||
return rep.add(entry, eng, "%s: the engine under test gave no answer (%s)" % (quantity, kind), why[:1500], None, False)
|
||
if fail_on_error and ans.get("error"):
|
||
return rep.add(entry, eng, quantity, None, None, False, note=why)
|
||
return rep.not_run(entry, eng, quantity, why)
|
||
|
||
|
||
def answered(rep, entry, eng):
|
||
"""The engine under test answered. Where it must refuse, that fails the run."""
|
||
if under_test(eng):
|
||
x = EXPECTED.get(entry, eng)
|
||
if x:
|
||
EXPECTED.answered.add((entry, eng))
|
||
if x["must"] == "refuse":
|
||
rep.add(entry, eng, "accepts a circuit it must refuse: " + x["reason"], True, False, False)
|
||
return False
|
||
return True
|
||
|
||
|
||
# ------------------------------------------------------------------ states
|
||
|
||
|
||
def load_state(path):
|
||
import numpy as np
|
||
|
||
return np.fromfile(path, dtype=np.complex128)
|
||
|
||
|
||
def exact_distance(ref, vec):
|
||
"""(strict, modulo a global phase, the phase) between the integer reference and a state."""
|
||
re = exact.ints_from_floats([float(x) for x in vec.real])
|
||
im = exact.ints_from_floats([float(x) for x in vec.imag])
|
||
d, dm, phase, _, _ = exact.distance(ref.re, ref.im, re, im)
|
||
return d, dm, phase
|
||
|
||
|
||
def float_distance(a, b):
|
||
"""(strict, modulo a global phase) between two double states, the sums taken exactly enough:
|
||
math.fsum over the squares."""
|
||
import numpy as np
|
||
|
||
def norm(v):
|
||
return math.sqrt(math.fsum((v.real**2).tolist()) + math.fsum((v.imag**2).tolist()))
|
||
|
||
d = norm(a - b)
|
||
# Turn b by the phase that best aligns it with a, then measure the difference itself:
|
||
# sqrt(|a|^2 + |b|^2 - 2|<b|a>|) would lose half its digits to cancellation.
|
||
ip = np.vdot(b, a)
|
||
turn = ip / abs(ip) if abs(ip) > 0 else 1.0
|
||
dm = norm(a - turn * b)
|
||
return d, dm
|
||
|
||
|
||
def ref_vector(ref):
|
||
import numpy as np
|
||
|
||
return np.array([complex(float(Fraction(r, exact.ONE)), float(Fraction(i, exact.ONE))) for r, i in zip(ref.re, ref.im)])
|
||
|
||
|
||
def state_tolerance(engine, ans, e):
|
||
gates = tol.gate_count(e["steps"])
|
||
n = e["n"]
|
||
if engine == "wai.exact":
|
||
return tol.fixed_point_bound(n, ans["ops"]), "the extension's published bound (quantum-sim.md section 5), summed over the op-log the engine ran"
|
||
if engine in ("wai.statevector",):
|
||
return tol.double_state(gates, n, width=3), "double precision, fusion width 3"
|
||
if engine in ("aer.statevector", "aer.qasm_import"):
|
||
g = max(gates, ans.get("settings", {}).get("gates", gates))
|
||
width = 5 if n >= 14 else 2
|
||
return tol.double_state(g, n, width=width), "double precision, fusion up to %d qubits from 14 qubits" % 5
|
||
if engine == "quimb.dense":
|
||
return tol.double_state(max(gates, ans.get("gates", gates)), n, width=3), "double precision, gate by gate"
|
||
if engine == "quimb.tensor_network":
|
||
w = ans.get("settings", {}).get("contraction_width") or n
|
||
return tol.contraction(ans.get("gates", gates) + n, min(float(w), 40.0)), "double precision, a contraction of width %.1f" % float(w)
|
||
if engine == "ddsim.statevector":
|
||
return tol.decision_diagram(gates, n), "decision-diagram edge weights merged within 2^-42"
|
||
if engine == "qsim.statevector":
|
||
return tol.single_state(gates, n, width=2), "single precision, fusion width 2"
|
||
raise KeyError(engine)
|
||
|
||
|
||
def run_oplog(n, ops):
|
||
"""The bit-exact method's op-log run in double precision: its ideal state, to a double's
|
||
rounding. Base gates are quantum-sim.md's (section 3.0): I X Y Z H S S† T T† P = 0..9, P(k)
|
||
the phase e^(2πi/2^k); qubit 0 is the lowest bit of the index."""
|
||
import numpy as np
|
||
|
||
r = 1 / math.sqrt(2)
|
||
fixed = {0: ((1, 0), (0, 1)), 1: ((0, 1), (1, 0)), 2: ((0, -1j), (1j, 0)), 3: ((1, 0), (0, -1)), 4: ((r, r), (r, -r)), 5: ((1, 0), (0, 1j)), 6: ((1, 0), (0, -1j)), 7: ((1, 0), (0, complex(r, r))), 8: ((1, 0), (0, complex(r, -r)))}
|
||
v = np.zeros(1 << n, dtype=np.complex128)
|
||
v[0] = 1
|
||
idx = np.arange(1 << n)
|
||
pairs = {}
|
||
for base, controls, target, param in ops:
|
||
m = ((1, 0), (0, np.exp(2j * math.pi / (1 << param)) if param > 2 else (-1 if param == 1 else 1j))) if base == 9 else fixed[base]
|
||
key = (tuple(controls), target)
|
||
if key not in pairs:
|
||
mask = 0
|
||
for c in controls:
|
||
mask |= 1 << c
|
||
lo = idx[((idx & mask) == mask) & ((idx >> target) & 1 == 0)]
|
||
pairs[key] = (lo, lo | (1 << target))
|
||
lo, hi = pairs[key]
|
||
a, b = v[lo], v[hi]
|
||
v[lo], v[hi] = m[0][0] * a + m[0][1] * b, m[1][0] * a + m[1][1] * b
|
||
return v
|
||
|
||
|
||
def oplog_check(rep, e, n, ops, want):
|
||
"""The op-log the bit-exact method ran is the circuit: the op-log's ideal state against the
|
||
circuit's, modulo the global phase the lowering drops. Within the double rounding of that run,
|
||
the distance the reader's rule moves angles (corpus.reader_class: a phase moved by δ turns
|
||
changes a state by at most 2πδ), and 2^-50 for the reference's conversion to double. The
|
||
referee then ties the integers to the op-log, so this ties them to the circuit."""
|
||
cls, moved = corpus.reader_class(e)
|
||
_, dm = float_distance(want, run_oplog(n, ops))
|
||
bound = tol.double_state(len(ops), n, width=1) + 2 * math.pi * float(moved) + 2.0**-50
|
||
note = "the reader moved its angles by %.3g of a turn in all, as its rule has it" % float(moved) if cls == "moved" else None
|
||
rep.add(e["id"], "wai.exact", "the op-log is the circuit: the op-log's ideal state against the circuit's, modulo a global phase", dm, bound, dm <= bound, note=note)
|
||
|
||
|
||
def referee_hash(n, ops, fixed, engine_hash):
|
||
"""The referee's integer simulator run on the engine's op-log must give the integers the
|
||
engine wrote, and the referee's BLAKE3 of them the hash the engine reported."""
|
||
ref = referee()
|
||
table = ref.derive_v2()[0]
|
||
oplist = [ref.gate(op, t, tuple(c), p) for op, c, t, p in ops]
|
||
amps = ref.simulate(n, oplist, table)
|
||
mine = [(int(a), int(b)) for a, b in fixed]
|
||
same = [tuple(x) for x in amps] == mine
|
||
digest = ref.statevector_hash(n, amps)
|
||
digest = digest.hex() if isinstance(digest, (bytes, bytearray)) else digest
|
||
if not same:
|
||
return {"match": False, "note": "the referee's state differs from the engine's"}
|
||
if digest != engine_hash:
|
||
return {"match": False, "note": "the referee's hash %s, the engine's %s" % (digest, engine_hash)}
|
||
return {"match": True}
|
||
|
||
|
||
def exact_answer(rep, e, n, ans, want, quantity="state"):
|
||
"""The bit-exact method's checks beside its distance: the op-log is the circuit, and the
|
||
referee reruns the op-log to the engine's integers and hash (to 14 qubits)."""
|
||
import numpy as np
|
||
|
||
fixed = np.load(ans["settings"]["fixed"])
|
||
oplog_check(rep, e, n, ans["ops"], want)
|
||
if n <= 14:
|
||
h = referee_hash(n, ans["ops"], fixed, ans["state_hash"])
|
||
rep.add(e["id"], "wai.exact", "state hash, recomputed by the referee from the op-log and from the integers written", h["match"], True, h["match"], note=h.get("note"))
|
||
return fixed
|
||
|
||
|
||
def reference_checks(e, ref, rep):
|
||
"""A closed form, where there is one, holds the integer reference itself."""
|
||
n = e["n"]
|
||
unit = "units of 2^-256"
|
||
if "qft_x" in e:
|
||
worst = max(abs(exact.qft_amplitude(n, e["qft_x"], y)[0] - ref.re[y]) + abs(exact.qft_amplitude(n, e["qft_x"], y)[1] - ref.im[y]) for y in range(1 << n))
|
||
rep.add(e["id"], "exact", "integer reference against the closed form, " + unit, worst, REFERENCE_UNITS, worst <= REFERENCE_UNITS)
|
||
if e["family"] == "ghz":
|
||
r = exact.rsqrt2()
|
||
worst = max(abs(ref.re[i] - (r if i in (0, (1 << n) - 1) else 0)) + abs(ref.im[i]) for i in range(1 << n))
|
||
rep.add(e["id"], "exact", "integer reference against the closed form, " + unit, worst, REFERENCE_UNITS, worst <= REFERENCE_UNITS)
|
||
if "grover" in e:
|
||
m, k = e["grover"]
|
||
p = ref.re[m] ** 2 + ref.im[m] ** 2
|
||
err = abs(p - exact.grover_law(n, k)) >> exact.PREC
|
||
rep.add(e["id"], "exact", "integer reference: P(marked) against sin²((2k+1)θ), computed in integers, " + unit, err, REFERENCE_UNITS, err <= REFERENCE_UNITS)
|
||
if "qpe" in e:
|
||
m, bits = e["qpe"]
|
||
i = m | (1 << bits)
|
||
err = abs(ref.re[i] ** 2 + ref.im[i] ** 2 - (1 << (2 * exact.PREC))) >> exact.PREC
|
||
rep.add(e["id"], "exact", "integer reference: P(m) against 1, the closed form, " + unit, err, REFERENCE_UNITS, err <= REFERENCE_UNITS)
|
||
if "qpe_third" in e:
|
||
bits = e["qpe_third"]
|
||
worst = 0
|
||
for y in range(1 << bits):
|
||
i = y | (1 << bits)
|
||
worst = max(worst, abs(ref.re[i] ** 2 + ref.im[i] ** 2 - exact.qpe_third_law(bits, y)) >> exact.PREC)
|
||
rep.add(e["id"], "exact", "integer reference: every outcome's probability against sin²(π2^tδ)/(2^2t sin²(πδ)), " + unit, worst, REFERENCE_UNITS, worst <= REFERENCE_UNITS)
|
||
|
||
|
||
def probe_states(e):
|
||
"""The circuit with its probed gate left out, and that gate's angle in radians."""
|
||
steps = [s for i, s in enumerate(e["steps"]) if i != e["probe"]]
|
||
return exact.run_unitary(e["n"], steps), abs(exact.float_radians(e["steps"][e["probe"]]["a"][0]))
|
||
|
||
|
||
def phase_held(rep, e, eng, ref, probe, re_ints, im_ints):
|
||
"""Whether an engine holds the probed gate or drops it: its distance alone cannot tell them
|
||
apart once the gate's effect is below the engine's bound. Exactly, in integers, modulo a global
|
||
phase: it holds the gate when its answer is within half the gate's effect of the ideal state, and
|
||
drops it when within half of the state without the gate. When it is within half of neither, its
|
||
own rounding is larger than the gate's effect, and the answer says nothing either way: that is
|
||
recorded and not judged. The share of the step it took (exact.held_share) is recorded beside."""
|
||
without, theta = probe
|
||
share = exact.held_share(ref, without, re_ints, im_ints)
|
||
step = exact.distance(ref.re, ref.im, without.re, without.im)[1]
|
||
to_ideal = exact.distance(ref.re, ref.im, re_ints, im_ints)[1]
|
||
to_without = exact.distance(without.re, without.im, re_ints, im_ints)[1]
|
||
verdict = "holds it" if to_ideal <= step / 2 else "drops it" if to_without <= step / 2 else None
|
||
q = "holds the phase of its small-angle gate: the share of the step from the state without that gate to the ideal (1 holds it, 0 drops it)"
|
||
if verdict is None:
|
||
c = rep.add(e["id"], eng, q, share, [0.5, 1.5], True, status="info", note="neither can be told: its distance from the ideal state, %.3g, is more than half the gate's effect, %.3g" % (to_ideal, step))
|
||
elif eng == "ddsim.statevector" and theta < DDSIM_PARAMETER_TOLERANCE:
|
||
c = rep.add(e["id"], eng, q, share, [0.5, 1.5], True, status="info", note="%s: the angle, %.3g rad, is below its parameter tolerance of %g rad, where it may read an angle as 0" % (verdict, theta, DDSIM_PARAMETER_TOLERANCE))
|
||
else:
|
||
c = rep.add(e["id"], eng, q, share, [0.5, 1.5], verdict == "holds it", note=verdict)
|
||
c["step"] = step
|
||
|
||
|
||
def extension_figure(rep, e, fixed):
|
||
"""quantum-sim.md section 5: P2(19) applied 2^18 times to |1> ends at modulus 1.0000189 and
|
||
within 10^-5 rad of the half turn. The modulus is stated to seven decimals."""
|
||
re, im = int(fixed[1][0]), int(fixed[1][1])
|
||
modulus = math.hypot(re, im) / (1 << 30)
|
||
off = abs(math.remainder(math.atan2(im, re) - math.pi, 2 * math.pi))
|
||
rep.add(e["id"], "wai.exact", "modulus of the half turn's amplitude against the extension's figure, 1.0000189 (section 5)", modulus, [1.00001885, 1.00001895], 1.00001885 <= modulus <= 1.00001895)
|
||
rep.add(e["id"], "wai.exact", "its angle from the half turn, in radians, which the extension states is within 1e-5", off, 1e-5, off <= 1e-5)
|
||
|
||
|
||
def check_state_entry(e, runner, rep, engines):
|
||
import numpy as np
|
||
|
||
n = e["n"]
|
||
ref = exact.run_unitary(n, e["steps"])
|
||
want = ref_vector(ref)
|
||
reference_checks(e, ref, rep)
|
||
strict = bool(e.get("phase_is_result"))
|
||
probe = probe_states(e) if "probe" in e else None
|
||
exact_ops = None
|
||
jobs = {}
|
||
for eng in engines:
|
||
if eng == "aer.qasm_import" and n > 14:
|
||
continue
|
||
if eng == "quimb.tensor_network" and n > 12:
|
||
continue
|
||
jobs[eng] = {"id": e["id"], "kind": "state", "n": n, "steps": e["steps"], "qasm": e["qasm"], "probe": e.get("probe")}
|
||
answers = run_all(runner, jobs)
|
||
quantity = "distance from the exact state" + (", with its phase" if strict else ", modulo a global phase")
|
||
for eng, ans in answers.items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "state", fail_on_error=True)
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
vec = load_state(ans["job"]["state_out"])
|
||
if len(vec) != 1 << n:
|
||
rep.add(e["id"], eng, "state length", len(vec), 1 << n, False)
|
||
continue
|
||
if eng == "wai.exact":
|
||
fixed = np.load(ans["settings"]["fixed"])
|
||
re = exact.ints_from_fixed([int(x) for x in fixed[:, 0]])
|
||
im = exact.ints_from_fixed([int(x) for x in fixed[:, 1]])
|
||
d, dm, phase, _, _ = exact.distance(ref.re, ref.im, re, im)
|
||
bound, why = state_tolerance(eng, ans, e)
|
||
measured = d if strict else dm
|
||
c = rep.add(e["id"], eng, quantity, measured, bound, measured <= bound, note=why)
|
||
c["strict"] = d
|
||
c["global_phase"] = phase
|
||
# The phase the method's result states: with it, the state is the reference's, which writes
|
||
# each gate as method statevector does, nothing fitted.
|
||
turn = complex(math.cos(ans.get("global_phase", 0.0)), math.sin(ans.get("global_phase", 0.0)))
|
||
stated = exact.distance(ref.re, ref.im, *[exact.ints_from_floats([float(x) for x in part]) for part in ((vec * turn).real, (vec * turn).imag)])[0]
|
||
rep.add(e["id"], eng, "distance from the exact state, with the global phase its result states", stated, bound + 8 * tol.U_DOUBLE, stated <= bound + 8 * tol.U_DOUBLE)
|
||
exact_ops = ans["ops"]
|
||
exact_answer(rep, e, n, ans, want)
|
||
if e.get("extension_figure"):
|
||
extension_figure(rep, e, fixed)
|
||
continue
|
||
bound, why = state_tolerance(eng, ans, e)
|
||
d, dm, phase = exact_distance(ref, vec)
|
||
measured = d if strict else dm
|
||
c = rep.add(e["id"], eng, quantity, measured, bound, measured <= bound, note=why)
|
||
c["strict"] = d
|
||
c["global_phase"] = phase
|
||
c["seconds"] = ans.get("seconds")
|
||
if ans.get("notes"):
|
||
c["adapter"] = ans["notes"]
|
||
if probe:
|
||
phase_held(rep, e, eng, ref, probe, exact.ints_from_floats([float(x) for x in vec.real]), exact.ints_from_floats([float(x) for x in vec.imag]))
|
||
if is_clifford(e) and e["expect"] == "exact_state":
|
||
check_stabilizer_entry(e, runner, rep)
|
||
# Through the engine's whole path, as a user asks: chosen amplitudes, method auto.
|
||
picks = sorted({0, (1 << n) - 1, 1 % (1 << n)} | {max(range(1 << n), key=lambda i: ref.re[i] ** 2 + ref.im[i] ** 2)})
|
||
bits = [format(i, "0%db" % n) for i in picks]
|
||
ans = runner.run("wai.auto", {"id": e["id"], "kind": "amplitudes", "n": n, "steps": e["steps"], "bitstrings": bits})
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], "wai.auto", ans, "chosen amplitudes through the job path")
|
||
return
|
||
planned = ans.get("planned")
|
||
# The bit-exact method lowers rz to a phase gate, and so on; its result states the phase that
|
||
# drops, and its amplitudes are held to the reference's with that phase, nothing fitted.
|
||
turn = 1
|
||
if planned == "exact":
|
||
stated_phase = ((ans.get("accuracy") or {}).get("global_phase") or {}).get("radians", 0.0)
|
||
turn = complex(math.cos(stated_phase), math.sin(stated_phase))
|
||
worst = 0.0
|
||
for b in bits:
|
||
a = complex(*ans["amplitudes"][b]) * turn
|
||
worst = max(worst, exact.amplitude_error((ref.re[int(b, 2)], ref.im[int(b, 2)]), a.real, a.imag))
|
||
if planned == "exact" and exact_ops is None:
|
||
# The bound is the op-log's; the bit-exact method gives it.
|
||
a2 = runner.run("wai.exact", {"id": e["id"], "kind": "state", "n": n, "steps": e["steps"]})
|
||
exact_ops = a2.get("ops") if a2.get("ok") else None
|
||
if planned == "exact" and exact_ops is not None:
|
||
bound = tol.fixed_point_bound(n, exact_ops) + 4 * tol.U_DOUBLE
|
||
else:
|
||
bound = tol.double_state(tol.gate_count(e["steps"]), n, width=3)
|
||
rep.add(e["id"], "wai.auto", "chosen amplitudes through the job path, worst error" + (", with the global phase the result states" if planned == "exact" else ""), worst, bound, worst <= bound, note="planned %s" % planned)
|
||
|
||
|
||
# ------------------------------------------------------------------ stabilizers
|
||
|
||
|
||
def check_stabilizer_entry(e, runner, rep):
|
||
n = e["n"]
|
||
tb = exact.Tableau(n)
|
||
for s in e["steps"]:
|
||
if "g" in s:
|
||
tb.gate(s["g"], s["q"])
|
||
rows, rank, x_rank = exact.reduced_group(tb.stabilizers(), n)
|
||
texts = [exact.pauli_text(r, n) for r in rows]
|
||
digest = hashlib.sha256("\n".join(texts).encode()).hexdigest()
|
||
rep.add(e["id"], "exact", "the integer tableau's group has full rank", rank, n, rank == n)
|
||
jobs = {"stim.tableau": {"id": e["id"], "kind": "stabilizers", "n": n, "steps": e["steps"]}, "wai.stabilizer": {"id": e["id"], "kind": "stabilizers", "n": n, "steps": e["steps"], "paulis": [t[1:] for t in texts]}}
|
||
if n <= 1000:
|
||
jobs["aer.stabilizer"] = {"id": e["id"], "kind": "stabilizers", "n": n, "steps": e["steps"]}
|
||
answers = run_all(runner, jobs)
|
||
for eng, ans in answers.items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "stabilizer group", fail_on_error=True)
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
if eng == "wai.stabilizer":
|
||
want = [-1 if t[0] == "-" else 1 for t in texts]
|
||
wrong = sum(1 for a, b in zip(ans["values"], want) if a != b)
|
||
c = rep.add(e["id"], eng, "generators of the reference group whose expectation is not their sign", wrong, 0, wrong == 0 and len(ans["values"]) == n)
|
||
c["seconds"] = ans.get("seconds")
|
||
rep.add(e["id"], eng, "outcome dimension against the rank of the group's X part", ans["outcome_dimension"], x_rank, ans["outcome_dimension"] == x_rank)
|
||
continue
|
||
theirs = [exact.row_from_text(s) for s in ans["stabilizers"]]
|
||
r2, rank2, _ = exact.reduced_group(theirs, n)
|
||
d2 = hashlib.sha256("\n".join(exact.pauli_text(r, n) for r in r2).encode()).hexdigest()
|
||
c = rep.add(e["id"], eng, "reduced stabilizer group equals the reference's", d2 == digest, True, d2 == digest and rank2 == n)
|
||
c["seconds"] = ans.get("seconds")
|
||
|
||
|
||
def fixed_bits(z_masks, n):
|
||
"""The bits whose value the affine space fixes: bit i is fixed when e_i lies in the span of
|
||
the Z rows' masks (GF(2) elimination)."""
|
||
basis = {}
|
||
for m in z_masks:
|
||
while m:
|
||
top = m.bit_length() - 1
|
||
if top not in basis:
|
||
basis[top] = m
|
||
break
|
||
m ^= basis[top]
|
||
out = set()
|
||
for i in range(n):
|
||
v = 1 << i
|
||
while v:
|
||
top = v.bit_length() - 1
|
||
if top not in basis:
|
||
break
|
||
v ^= basis[top]
|
||
if v == 0:
|
||
out.add(i)
|
||
return out
|
||
|
||
|
||
def check_stabilizer_counts(e, runner, rep, alpha):
|
||
"""Outcomes of a Clifford circuit measured in Z: uniform on an affine space of dimension x_rank.
|
||
Every shot must lie in it; no outcome may come up more often than its probability 2^-x_rank
|
||
allows (by Bernstein, over all 2^x_rank of them); and each bit the space leaves free must read
|
||
1 half the time. A sampler confined to a smaller space that leaves every bit free is not seen at
|
||
this width; the README says so."""
|
||
n = e["n"]
|
||
tb = exact.Tableau(n)
|
||
gates = [s for s in e["steps"] if "g" in s]
|
||
for s in gates:
|
||
tb.gate(s["g"], s["q"])
|
||
rows, _, x_rank = exact.reduced_group(tb.stabilizers(), n)
|
||
z_rows = [r for r in rows if r[0] == 0]
|
||
free = [i for i in range(n) if i not in fixed_bits([r[1] for r in z_rows], n)]
|
||
shots = e["shots"]
|
||
p = 2.0**-x_rank
|
||
jobs = {eng: {"id": e["id"], "kind": "counts", "n": n, "bits": e["bits"], "steps": e["steps"], "shots": shots, "seed": 7} for eng in ("wai.stabilizer", "stim.tableau", "aer.stabilizer")}
|
||
for eng, ans in run_all(runner, jobs).items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "outcomes inside the stabilizer's affine space")
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
counts = ans["counts"]
|
||
got = sum(counts.values())
|
||
rep.add(e["id"], eng, "shots returned", got, shots, got == shots)
|
||
outside = 0
|
||
for k, v in counts.items():
|
||
x = int(k, 2)
|
||
for zr in z_rows:
|
||
if ((zr[1] & x).bit_count() + zr[2]) % 2:
|
||
outside += v
|
||
break
|
||
rep.add(e["id"], eng, "shots outside the affine space the stabilizers fix (dimension %d)" % x_rank, outside, 0, outside == 0)
|
||
a = alpha / 3 / 2
|
||
t = tol.bernstein(shots, p, a / 2.0**x_rank)
|
||
most = max(counts.values(), default=0)
|
||
rep.add(e["id"], eng, "the most frequent outcome's count above shots·2^-%d, in units of its Bernstein bound over all 2^%d outcomes" % (x_rank, x_rank), (most - shots * p) / t, 1.0, most - shots * p <= t)
|
||
worst = 0.0
|
||
bad = []
|
||
for i in free:
|
||
ones = sum(v for k, v in counts.items() if int(k, 2) >> i & 1)
|
||
tb_ = tol.bernstein(shots, 0.5, a / max(len(free), 1))
|
||
worst = max(worst, abs(ones - shots / 2) / tb_)
|
||
if abs(ones - shots / 2) > tb_:
|
||
bad.append("bit %d: %d ones of %d" % (i, ones, shots))
|
||
rep.add(e["id"], eng, "each of the %d bits the space leaves free against 1/2, largest deviation in units of its Bernstein bound" % len(free), worst, 1.0, not bad, note="; ".join(bad[:4]) if bad else None)
|
||
rep.add(e["id"], eng, "distinct outcomes, of %d shots over 2^%d" % (shots, x_rank), len(counts), None, True, status="info")
|
||
|
||
|
||
# ------------------------------------------------------------------ sampled distributions
|
||
|
||
|
||
def check_counts_entry(e, runner, rep, plan):
|
||
dist = exact.distribution(e["n"], e["bits"], e["steps"])
|
||
engines = list(COUNT_ENGINES)
|
||
if is_clifford(e):
|
||
engines += ["wai.stabilizer", "stim.tableau"]
|
||
jobs = {eng: {"id": e["id"], "kind": "counts", "n": e["n"], "bits": e["bits"], "steps": e["steps"], "shots": e["shots"], "seed": 11} for eng in engines}
|
||
for eng, ans in run_all(runner, jobs).items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "counts", fail_on_error=True)
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
plan.append((e, eng, ans, dist))
|
||
|
||
|
||
def judge_counts(plan, rep, alpha):
|
||
"""Every outcome's count against its exact probability, by Bernstein's inequality, and the
|
||
whole distribution at once against the exact one, in L1 distance (tolerance.py); each test
|
||
at a share of `alpha`, so that a correct engine fails them with probability at most alpha."""
|
||
tests = sum(len(set(d) | set(a["counts"])) + 1 for _, _, a, d in plan)
|
||
a = alpha / max(tests, 1)
|
||
for e, eng, ans, dist in plan:
|
||
shots = sum(ans["counts"].values())
|
||
if shots != e["shots"]:
|
||
rep.add(e["id"], eng, "shots returned", shots, e["shots"], False)
|
||
continue
|
||
worst = 0.0
|
||
bad = []
|
||
for k in sorted(set(dist) | set(ans["counts"])):
|
||
p = float(dist.get(k, 0))
|
||
x = ans["counts"].get(k, 0)
|
||
if p == 0:
|
||
if x:
|
||
bad.append("%s: %d shots, probability 0" % (k, x))
|
||
continue
|
||
t = tol.bernstein(shots, p, a)
|
||
dev = abs(x - shots * p)
|
||
worst = max(worst, dev / t)
|
||
if dev > t:
|
||
bad.append("%s: %d shots, expected %.1f ± %.1f" % (k, x, shots * p, t))
|
||
c = rep.add(e["id"], eng, "largest deviation of an outcome's count, in units of its Bernstein bound", worst, 1.0, not bad, note="; ".join(bad[:4]) if bad else None)
|
||
if ans.get("planned"):
|
||
c["planned"] = ans["planned"]
|
||
support = sum(1 for v in dist.values() if v > 0)
|
||
l1 = math.fsum(abs(ans["counts"].get(k, 0) / shots - float(dist.get(k, 0))) for k in set(dist) | set(ans["counts"]))
|
||
eps = tol.l1_distance(shots, support + 1, a)
|
||
rep.add(e["id"], eng, "L1 distance from the exact distribution, Σ|count/shots − p| over every outcome (twice the total variation), against its bound", l1, eps, l1 <= eps)
|
||
|
||
|
||
# ------------------------------------------------------------------ wider circuits
|
||
|
||
|
||
def check_mirror_entry(e, runner, rep):
|
||
import numpy as np
|
||
|
||
n = e["n"]
|
||
engines = ["wai.exact", "wai.statevector", "aer.statevector", "quimb.dense", "ddsim.statevector", "qsim.statevector"]
|
||
jobs = {eng: {"id": e["id"], "kind": "state", "n": n, "steps": e["steps"]} for eng in engines}
|
||
for eng, ans in run_all(runner, jobs).items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "distance from |0…0>")
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
if eng == "wai.exact":
|
||
fixed = np.load(ans["settings"]["fixed"])
|
||
v = fixed[:, 0].astype(np.float64) / (1 << 30) + 1j * fixed[:, 1].astype(np.float64) / (1 << 30)
|
||
else:
|
||
v = load_state(ans["job"]["state_out"])
|
||
target = np.zeros_like(v)
|
||
target[0] = 1
|
||
d, dm = float_distance(v, target)
|
||
bound, why = state_tolerance(eng, ans, e)
|
||
c = rep.add(e["id"], eng, "distance from |0…0>, modulo a global phase", dm, bound, dm <= bound, note=why)
|
||
c["strict"] = d
|
||
c["seconds"] = ans.get("seconds")
|
||
if eng == "wai.exact":
|
||
exact_answer(rep, e, n, ans, target)
|
||
|
||
|
||
def check_pairwise_entry(e, runner, rep):
|
||
import numpy as np
|
||
|
||
n = e["n"]
|
||
engines = ["wai.statevector", "aer.statevector", "quimb.dense", "ddsim.statevector", "qsim.statevector"]
|
||
if corpus.reader_class(e)[0] == "exact":
|
||
engines.insert(0, "wai.exact")
|
||
jobs = {eng: {"id": e["id"], "kind": "state", "n": n, "steps": e["steps"]} for eng in engines}
|
||
got = {}
|
||
for eng, ans in run_all(runner, jobs).items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "state, against the other engines")
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
if eng == "wai.exact":
|
||
fixed = np.load(ans["settings"]["fixed"])
|
||
v = fixed[:, 0].astype(np.float64) / (1 << 30) + 1j * fixed[:, 1].astype(np.float64) / (1 << 30)
|
||
else:
|
||
v = load_state(ans["job"]["state_out"])
|
||
got[eng] = (v, state_tolerance(eng, ans, e)[0], ans)
|
||
names = sorted(got)
|
||
for i, a in enumerate(names):
|
||
for b in names[i + 1:]:
|
||
_, dm = float_distance(got[a][0], got[b][0])
|
||
bound = got[a][1] + got[b][1]
|
||
rep.add(e["id"], "%s ~ %s" % (a, b), "distance between the two states, modulo a global phase", dm, bound, dm <= bound)
|
||
if "qft_x" in e:
|
||
r = corpus.Stream("pick/" + e["id"])
|
||
ys = [r.below(1 << n) for _ in range(64)]
|
||
for eng in names:
|
||
v = got[eng][0]
|
||
worst = max(exact.amplitude_error(exact.qft_amplitude(n, e["qft_x"], y), float(v[y].real), float(v[y].imag)) for y in ys)
|
||
rep.add(e["id"], eng, "64 amplitudes against the closed form, worst error, modulo nothing", worst, got[eng][1], worst <= got[eng][1])
|
||
|
||
|
||
def wide_amplitudes(e):
|
||
n = e["n"]
|
||
r = corpus.Stream("wide/" + e["id"])
|
||
if e.get("wide_kind") == "ghz":
|
||
bits = ["0" * n, "1" * n, "".join(str(r.below(2)) for _ in range(n))]
|
||
ref = {bits[0]: (exact.rsqrt2(), 0), bits[1]: (exact.rsqrt2(), 0), bits[2]: (0, 0)}
|
||
return bits, ref
|
||
if e.get("wide_kind") == "qft":
|
||
ys = [r.below(1 << n) for _ in range(6)]
|
||
bits = [format(y, "0%db" % n) for y in ys]
|
||
return bits, {b: exact.qft_amplitude(n, e["qft_x"], int(b, 2)) for b in bits}
|
||
if e["expect"] == "wide_mirror":
|
||
b = "0" * n
|
||
return [b], {b: (exact.ONE, 0)}
|
||
bits = ["".join(str(r.below(2)) for _ in range(n)) for _ in range(8)]
|
||
return bits, None
|
||
|
||
|
||
def mps_bound(eng, ans, e):
|
||
"""tolerance.mps for each engine's matrix product state, charged what that engine reports it
|
||
truncated: the engine under test its summed discarded weight over its truncations, the
|
||
simulator with a log its every truncation's weight and its two-site updates, and the library
|
||
with an absolute cutoff the most that cutoff drops at each update (engines.py)."""
|
||
steps = tol.svd_steps(e["steps"])
|
||
acc = ans.get("accuracy") or {}
|
||
if eng == "wai.mps":
|
||
chi = acc.get("max_bond_used") or acc.get("max_bond") or 64
|
||
trunc = tol.truncation_from(total=acc.get("discarded_weight", 0.0), count=acc.get("truncations", 0))
|
||
why = "%d two-site updates at bond %d; truncation from its reported discarded weight" % (steps, chi)
|
||
elif eng == "aer.mps":
|
||
log = ans["settings"]["mps_log"]
|
||
steps, chi = log["updates"], log["max_bond"]
|
||
trunc = tol.truncation_from(weights=log["discarded"])
|
||
why = "%d two-site updates at bond %d, and %d truncations, from its log" % (steps, chi, len(log["discarded"]))
|
||
else:
|
||
s = ans["settings"]
|
||
chi = s.get("max_bond_used") or 1
|
||
trunc = 2 * steps * math.sqrt(s["cutoff"])
|
||
why = "%d two-site updates at bond %d; its cutoff drops at most %g of weight at each" % (steps, chi, s["cutoff"])
|
||
return tol.mps(steps, chi, trunc), why
|
||
|
||
|
||
def check_wide_entry(e, runner, rep):
|
||
n = e["n"]
|
||
bits, ref = wide_amplitudes(e)
|
||
engines = ["wai.mps", "wai.tensor_network", "aer.mps", "quimb.mps", "quimb.tensor_network"]
|
||
if n <= 26:
|
||
engines += ["wai.statevector", "aer.statevector"]
|
||
engines = e.get("only_engines") or engines
|
||
jobs = {eng: {"id": e["id"], "kind": "amplitudes", "n": n, "steps": e["steps"], "bitstrings": bits, "memory_gb": 24, "max_bond": e.get("max_bond")} for eng in engines}
|
||
got = {}
|
||
gates = tol.gate_count(e["steps"])
|
||
smallest = min((math.hypot(*ref[b]) / exact.ONE for b in bits if ref and any(ref[b])), default=None) if ref else None
|
||
for eng, ans in run_all(runner, jobs).items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "amplitudes")
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
if eng in ("wai.mps", "aer.mps", "quimb.mps"):
|
||
bound, why = mps_bound(eng, ans, e)
|
||
elif eng in ("wai.tensor_network", "quimb.tensor_network"):
|
||
if "contraction_flops" in ans:
|
||
w = math.log2(max(ans["contraction_flops"], 2.0))
|
||
else:
|
||
w = float(ans.get("settings", {}).get("contraction_width") or n)
|
||
bound = tol.contraction(gates + n, min(w, 40.0))
|
||
why = "contraction of width %.1f" % w
|
||
else:
|
||
bound = tol.double_state(gates, n, width=5)
|
||
why = "double precision state vector"
|
||
got[eng] = (ans["amplitudes"], bound, ans)
|
||
if ref is not None:
|
||
worst = max(exact.amplitude_error(ref[b], *ans["amplitudes"][b]) for b in bits)
|
||
# A bound as large as half the smallest amplitude checked cannot see a wrong phase on it.
|
||
if smallest is not None and bound >= smallest / 2:
|
||
c = rep.add(e["id"], eng, "amplitudes against the closed form, worst error", worst, bound, True, status="info", note=why + "; this bound is not below half the amplitudes it checks (%.3g), so it cannot tell a wrong phase" % smallest)
|
||
else:
|
||
c = rep.add(e["id"], eng, "amplitudes against the closed form, worst error", worst, bound, worst <= bound, note=why)
|
||
c["seconds"] = ans.get("seconds")
|
||
if ans.get("accuracy"):
|
||
c["accuracy"] = ans["accuracy"]
|
||
if ref is None:
|
||
names = sorted(got)
|
||
for i, a in enumerate(names):
|
||
for b in names[i + 1:]:
|
||
worst = max(abs(complex(*got[a][0][x]) - complex(*got[b][0][x])) for x in bits)
|
||
bound = got[a][1] + got[b][1]
|
||
rep.add(e["id"], "%s ~ %s" % (a, b), "chosen amplitudes, worst difference", worst, bound, worst <= bound)
|
||
|
||
|
||
def check_expectation_entry(e, runner, rep):
|
||
kind, p, k = e["closed"]
|
||
p = Fraction(p)
|
||
if kind == "depol1":
|
||
ref = (-1) ** k * (1 - 4 * p / 3) ** k
|
||
else:
|
||
ref = (1 - 16 * p / 15) ** k
|
||
jobs = {eng: {"id": e["id"], "kind": "expectation", "n": e["n"], "steps": e["steps"], "noise": e["noise"], "observable": e["observable"]} for eng in ("wai.density_matrix", "aer.density_matrix")}
|
||
for eng, ans in run_all(runner, jobs).items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, e["id"], eng, ans, "expectation under noise")
|
||
continue
|
||
if not answered(rep, e["id"], eng):
|
||
continue
|
||
d = abs(Fraction(ans["value"]) - ref)
|
||
bound = tol.density(k, e["n"])
|
||
rep.add(e["id"], eng, "<%s> against the closed form" % e["observable"], float(d), bound, float(d) <= bound)
|
||
|
||
|
||
# ------------------------------------------------------------------ error correction
|
||
|
||
|
||
def qec_text(q, runner, rep):
|
||
"""The committed circuit, which the pinned generator must still write byte for byte."""
|
||
text = open(os.path.join(ROOT, "corpus", "qec", q["id"] + ".circuit")).read()
|
||
ans = runner.run("stim.generate", dict(q, kind="generate"))
|
||
same = ans.get("ok") and ans.get("text") == text
|
||
rep.add(q["id"], "stim.generate", "the pinned generator writes the committed circuit, byte for byte", bool(same), True, bool(same), note=None if same else (ans.get("error") or "the text differs"))
|
||
return text
|
||
|
||
|
||
def merged(mechs):
|
||
out = {}
|
||
for p, ds, ls in mechs:
|
||
key = (tuple(ds), tuple(ls))
|
||
q = out.get(key, 0.0)
|
||
out[key] = q + p - 2 * q * p
|
||
return out
|
||
|
||
|
||
def check_qec_entry(q, text, runner, rep, qec_plan):
|
||
jobs = {eng: {"id": q["id"], "kind": "qec_model", "text": text} for eng in ("wai.qec", "stim.tableau")}
|
||
models = run_all(runner, jobs)
|
||
a, b = models["wai.qec"], models["stim.tableau"]
|
||
if not a.get("ok"):
|
||
no_answer(rep, q["id"], "wai.qec", a, "detector error model")
|
||
if not b.get("ok"):
|
||
rep.not_run(q["id"], "stim.tableau", "detector error model", str(words(b))[:300])
|
||
return
|
||
if not answered(rep, q["id"], "wai.qec"):
|
||
return
|
||
if not b.get("ok"):
|
||
# A channel the other engine gives no exact model of (a general Pauli channel): its
|
||
# sampler still holds the engine under test's model, by the rates below.
|
||
rep.not_run(q["id"], "stim.tableau", "detector error model", "gives no model of this circuit, which the engine under test models: %s" % str(words(b))[:300])
|
||
shots = 200000 if q["tier"] == "ci" else 1000000
|
||
sj = {eng: {"id": q["id"], "kind": "qec_sample", "text": text, "shots": shots, "seed": 5} for eng in ("wai.qec", "stim.tableau")}
|
||
qec_plan.append((q, merged(a["mechanisms"]), a["detectors"], a["observables"], run_all(runner, sj), shots))
|
||
return
|
||
rep.add(q["id"], "wai.qec ~ stim.tableau", "detectors and observables", [a["detectors"], a["observables"]], [b["detectors"], b["observables"]], [a["detectors"], a["observables"]] == [b["detectors"], b["observables"]])
|
||
ma, mb = merged(a["mechanisms"]), merged(b["mechanisms"])
|
||
only_a = sorted(set(ma) - set(mb))
|
||
only_b = sorted(set(mb) - set(ma))
|
||
sites = text_noise_sites(text)
|
||
bound = tol.dem(sites)
|
||
worst = max([abs(ma[k] - mb[k]) for k in set(ma) & set(mb)] or [0.0])
|
||
ok = not only_a and not only_b and worst <= bound
|
||
note = None
|
||
if only_a or only_b:
|
||
note = "symptom sets only in the engine under test: %d, only in the other: %d (first: %s / %s)" % (len(only_a), len(only_b), only_a[:1], only_b[:1])
|
||
c = rep.add(q["id"], "wai.qec ~ stim.tableau", "mechanisms: same symptom sets, probabilities within the bound (largest difference)", worst, bound, ok, note=note)
|
||
c["mechanisms"] = [len(ma), len(mb)]
|
||
c["seconds"] = [a.get("seconds"), b.get("seconds")]
|
||
shots = 200000 if q["tier"] == "ci" else 1000000
|
||
sj = {eng: {"id": q["id"], "kind": "qec_sample", "text": text, "shots": shots, "seed": 5} for eng in ("wai.qec", "stim.tableau")}
|
||
samples = run_all(runner, sj)
|
||
qec_plan.append((q, ma, a["detectors"], a["observables"], samples, shots))
|
||
|
||
|
||
def text_noise_sites(text):
|
||
"""How many noise channels a circuit applies, its blocks unrolled: an upper bound on how many
|
||
channels one mechanism can merge."""
|
||
import re
|
||
|
||
total = 0
|
||
stack = [1]
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
if s.startswith("REPEAT"):
|
||
stack.append(stack[-1] * int(s.split()[1]))
|
||
continue
|
||
if s == "}":
|
||
stack.pop()
|
||
continue
|
||
m = re.match(r"([A-Z_0-9]+)(\[[^\]]*\])?\(([^)]*)\)\s*(.*)", s)
|
||
if m and (m.group(1).endswith("ERROR") or m.group(1).startswith(("DEPOLARIZE", "PAULI_CHANNEL")) or m.group(1) in ("E", "M", "MX", "MR", "MRX", "MY")):
|
||
total += stack[-1] * max(len(m.group(4).split()), 1)
|
||
return total
|
||
|
||
|
||
def judge_qec(qec_plan, rep, alpha):
|
||
"""Each detector's and observable's firing count against the rate the model gives,
|
||
by Bernstein's inequality at a share of `alpha`."""
|
||
tests = sum(d + o for _, _, d, o, _, _ in qec_plan) * 2
|
||
a = alpha / max(tests, 1)
|
||
for q, model, nd, no, samples, shots in qec_plan:
|
||
# The rate at which a detector fires: an odd number of its mechanisms, each independent.
|
||
prod_d = [1.0] * nd
|
||
prod_o = [1.0] * no
|
||
for (ds, ls), p in model.items():
|
||
for d in ds:
|
||
prod_d[d] *= 1 - 2 * p
|
||
for o in ls:
|
||
prod_o[o] *= 1 - 2 * p
|
||
rates = [(1 - x) / 2 for x in prod_d] + [(1 - x) / 2 for x in prod_o]
|
||
for eng, ans in samples.items():
|
||
if not ans.get("ok"):
|
||
no_answer(rep, q["id"], eng, ans, "detection rates")
|
||
continue
|
||
shape = [len(ans.get("detectors") or []), len(ans.get("observables") or []), ans.get("shots")]
|
||
if shape != [nd, no, shots]:
|
||
rep.add(q["id"], eng, "detectors, observables and shots returned", shape, [nd, no, shots], False)
|
||
continue
|
||
counts = ans["detectors"] + ans["observables"]
|
||
worst = 0.0
|
||
bad = []
|
||
for i, (x, r) in enumerate(zip(counts, rates)):
|
||
if r == 0:
|
||
if x:
|
||
bad.append("%d fired %d times at rate 0" % (i, x))
|
||
continue
|
||
t = tol.bernstein(shots, r, a)
|
||
dev = abs(x - shots * r)
|
||
worst = max(worst, dev / t)
|
||
if dev > t:
|
||
bad.append("%s %d: %d, expected %.1f ± %.1f" % ("D" if i < nd else "L", i if i < nd else i - nd, x, shots * r, t))
|
||
c = rep.add(q["id"], eng, "detection and flip rates against the model, largest deviation in units of its Bernstein bound", worst, 1.0, not bad, note="; ".join(bad[:3]) if bad else None)
|
||
c["seconds"] = ans.get("seconds")
|
||
|
||
|
||
# ------------------------------------------------------------------ the run
|
||
|
||
|
||
def run_all(runner, jobs):
|
||
with ThreadPoolExecutor(max_workers=runner.jobs) as ex:
|
||
futs = {eng: ex.submit(runner.run, eng, job) for eng, job in jobs.items()}
|
||
return {eng: f.result() for eng, f in futs.items()}
|
||
|
||
|
||
def host():
|
||
info = {"system": platform.system(), "machine": platform.machine(), "release": platform.release()}
|
||
try:
|
||
if platform.system() == "Darwin":
|
||
info["cpu"] = subprocess.run(["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True).stdout.strip()
|
||
info["memory_bytes"] = int(subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True).stdout)
|
||
info["os"] = "macOS " + platform.mac_ver()[0]
|
||
else:
|
||
for line in open("/proc/cpuinfo"):
|
||
if line.startswith("model name"):
|
||
info["cpu"] = line.split(":", 1)[1].strip()
|
||
break
|
||
for line in open("/proc/meminfo"):
|
||
if line.startswith("MemTotal"):
|
||
info["memory_bytes"] = int(line.split()[1]) * 1024
|
||
try:
|
||
for line in open("/etc/os-release"):
|
||
if line.startswith("PRETTY_NAME="):
|
||
info["os"] = line.split("=", 1)[1].strip().strip('"')
|
||
except OSError:
|
||
pass
|
||
info["logical_cpus"] = os.cpu_count()
|
||
info["load_average"] = list(os.getloadavg())
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
return info
|
||
|
||
|
||
def run_reproducer(path):
|
||
"""A committed reproducer of another engine's disagreement: it prints one JSON object per
|
||
line, each a claim, what it saw, and whether the claim holds."""
|
||
out = subprocess.run([sys.executable, "-I", os.path.join(ROOT, path)], capture_output=True, text=True, timeout=600)
|
||
claims = []
|
||
for line in out.stdout.splitlines():
|
||
line = line.strip()
|
||
if line.startswith("{"):
|
||
claims.append(json.loads(line))
|
||
if out.returncode != 0 and not claims:
|
||
claims.append({"claim": "the reproducer runs", "observed": out.stderr[-500:], "holds": False})
|
||
return claims
|
||
|
||
|
||
def apply_known(rep, entries_run):
|
||
"""A disagreement of another engine with the exact reference, found and examined before, is
|
||
listed in corpus/known-disagreements.json with what was found and a reproducer committed beside
|
||
it. A failing check that matches one is reported as `known` rather than as a failure of this
|
||
run, and the reproducer's claims are rerun and recorded; one listed that no longer shows is
|
||
reported too. Only other engines may be listed: a failure of the engine under test is always a
|
||
failure."""
|
||
known = json.load(open(os.path.join(ROOT, "corpus", "known-disagreements.json")))["disagreements"]
|
||
seen = set()
|
||
for i, k in enumerate(known):
|
||
if k["engine"].startswith("wai."):
|
||
rep.add(k["entry"], k["engine"], "a known disagreement listed for the engine under test", True, False, False, note="only other engines may be listed")
|
||
seen.add(i)
|
||
if not os.path.exists(os.path.join(ROOT, k.get("reproducer", ""))) or not k.get("reproducer"):
|
||
rep.add(k["entry"], k["engine"], "a known disagreement has a reproducer committed beside it", k.get("reproducer"), True, False)
|
||
seen.add(i)
|
||
for c in rep.checks:
|
||
if c["status"] != "fail":
|
||
continue
|
||
for i, k in enumerate(known):
|
||
quantities = k["quantity"] if isinstance(k["quantity"], list) else [k["quantity"]]
|
||
if i not in seen and c["entry"] == k["entry"] and c["engine"] == k["engine"] and any(x in c["quantity"] for x in quantities):
|
||
c["status"] = "known"
|
||
c["note"] = ((c.get("note") or "") + " | known: " + k["finding"]).strip(" |")
|
||
c["right"] = k["right"]
|
||
k["_matched"] = True
|
||
done = set()
|
||
for k in known:
|
||
if k["entry"] not in entries_run or k["engine"].startswith("wai."):
|
||
continue
|
||
if not k.get("_matched"):
|
||
rep.add(k["entry"], k["engine"], "a listed disagreement no longer shows", False, True, True, status="info", note=k["finding"])
|
||
if k.get("reproducer") and k["reproducer"] not in done and os.path.exists(os.path.join(ROOT, k["reproducer"])):
|
||
done.add(k["reproducer"])
|
||
for cl in run_reproducer(k["reproducer"]):
|
||
rep.add(k["entry"], k["engine"], "reproducer (%s): %s" % (k["reproducer"], cl["claim"]), cl["observed"], cl.get("expected"), True, status="known" if cl["holds"] else "info", note=None if cl["holds"] else "the claim no longer holds")
|
||
|
||
|
||
def check_expected(rep, entries_run):
|
||
"""The expected-refusals file is what corpus.py's rule and list give, and a listed refusal that
|
||
no longer comes is reported (a `may` that the engine now answers)."""
|
||
want = corpus.expected_refusals()
|
||
rep.add("corpus", "corpus.py", "corpus/expected-refusals.json is the reader's rule and corpus.py's list", EXPECTED.rows == want, True, EXPECTED.rows == want)
|
||
for entry, eng in sorted(EXPECTED.answered):
|
||
x = EXPECTED.get(entry, eng)
|
||
if x["must"] == "may" and entry in entries_run:
|
||
rep.add(entry, eng, "a listed refusal no longer comes: the engine answered", True, False, True, status="info", note=x["reason"])
|
||
|
||
|
||
def phase_table_check(rep):
|
||
"""quantum-sim.md section 3.2's table: P(k) for k = 1..32, cos and sin of 2π/2^k rounded to the
|
||
nearest 2^-30. The integer reference must give the referee's table, entry for entry."""
|
||
table = referee().derive_v2()[0]
|
||
half = 1 << (exact.PREC - 31)
|
||
wrong = []
|
||
for k in range(1, 33):
|
||
c, s = exact.cos_sin("t:1/%d" % k)
|
||
mine = ((c + half) >> (exact.PREC - 30), (s + half) >> (exact.PREC - 30))
|
||
if mine != tuple(table[k]):
|
||
wrong.append(k)
|
||
rep.add("corpus", "exact", "the extension's phase table P(1..32): the integer reference's cos and sin, rounded to the nearest 2^-30, against the referee's", wrong, [], not wrong)
|
||
|
||
|
||
def manifest_check(es, rep):
|
||
"""The generator still gives the bytes the manifest pins."""
|
||
m = json.load(open(os.path.join(ROOT, "corpus", "manifest.json")))
|
||
pinned = {r["id"]: r.get("sha256") for r in m["circuits"]}
|
||
drift = [e["id"] for e in es if pinned.get(e["id"]) != hashlib.sha256(e["qasm"].encode()).hexdigest()]
|
||
for q in corpus.qec_entries():
|
||
text = open(os.path.join(ROOT, "corpus", "qec", q["id"] + ".circuit")).read()
|
||
if pinned.get(q["id"]) != hashlib.sha256(text.encode()).hexdigest():
|
||
drift.append(q["id"])
|
||
rep.add("corpus", "corpus.py", "circuits whose text differs from the manifest", drift, [], not drift)
|
||
for e in es:
|
||
p = os.path.join(ROOT, "corpus", "qasm", e["id"] + ".qasm")
|
||
if os.path.exists(p) and open(p).read() != e["qasm"]:
|
||
rep.add(e["id"], "corpus.py", "the committed program equals the generator's", False, True, False)
|
||
|
||
|
||
CI_HAND = ("qec_inverted_and_tagged", "qec_correlated", "qec_refused_random_detector")
|
||
|
||
|
||
def main():
|
||
global EXPECTED
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--tier", default="ci", choices=["ci", "full"])
|
||
ap.add_argument("--out")
|
||
ap.add_argument("--jobs", type=int, default=2)
|
||
ap.add_argument("--timeout", type=float, default=900)
|
||
ap.add_argument("--inject", choices=sorted(FAULTS))
|
||
ap.add_argument("--only")
|
||
ap.add_argument("--keep", action="store_true")
|
||
a = ap.parse_args()
|
||
if not os.environ.get("WAI_CROSS_SIMULATOR"):
|
||
print("compare.py: WAI_CROSS_SIMULATOR must name the engine's driver (reproduce.sh builds it)", file=sys.stderr)
|
||
sys.exit(2)
|
||
work = tempfile.mkdtemp(prefix="qcorpus-")
|
||
runner = Runner(work, sys.executable, a.timeout, a.inject)
|
||
runner.jobs = a.jobs
|
||
rep = Report()
|
||
EXPECTED = Expected()
|
||
started = time.time()
|
||
es = corpus.build()
|
||
manifest_check(es, rep)
|
||
phase_table_check(rep)
|
||
if a.tier == "ci":
|
||
es = [e for e in es if e["tier"] == "ci"]
|
||
only = set(a.only.split(",")) if a.only else None
|
||
if only:
|
||
es = [e for e in es if e["id"] in only]
|
||
count_plan = []
|
||
stab_counts = sum(1 for e in es if e["expect"] == "stabilizer_counts")
|
||
for e in es:
|
||
print("·", e["id"], file=sys.stderr, flush=True)
|
||
x = e["expect"]
|
||
if x in ("exact_state", "known_state"):
|
||
check_state_entry(e, runner, rep, e.get("engines") or STATE_ENGINES)
|
||
elif x == "distribution":
|
||
check_counts_entry(e, runner, rep, count_plan)
|
||
elif x == "stabilizer":
|
||
check_stabilizer_entry(e, runner, rep)
|
||
elif x == "stabilizer_counts":
|
||
check_stabilizer_counts(e, runner, rep, FAMILY_WISE_ALPHA / 3 / max(stab_counts, 1))
|
||
elif x == "mirror":
|
||
check_mirror_entry(e, runner, rep)
|
||
elif x == "pairwise":
|
||
check_pairwise_entry(e, runner, rep)
|
||
elif x in ("amplitudes", "wide_pairwise", "wide_mirror"):
|
||
check_wide_entry(e, runner, rep)
|
||
elif x == "expectation":
|
||
check_expectation_entry(e, runner, rep)
|
||
judge_counts(count_plan, rep, FAMILY_WISE_ALPHA / 3)
|
||
qplan = []
|
||
qs = [q for q in corpus.qec_entries() if a.tier == "full" or q["tier"] == "ci"]
|
||
hand = json.load(open(os.path.join(ROOT, "corpus", "qec-hand-written.json")))
|
||
if only:
|
||
qs = [q for q in qs if q["id"] in only]
|
||
hand = {k: v for k, v in hand.items() if k in only}
|
||
for q in qs:
|
||
print("·", q["id"], file=sys.stderr, flush=True)
|
||
text = qec_text(q, runner, rep)
|
||
check_qec_entry(q, text, runner, rep, qplan)
|
||
if q["code"] == "surface_code:rotated_memory_z" and os.environ.get("WAI_QUANTUM"):
|
||
mine = subprocess.run([os.environ["WAI_QUANTUM"], "lab", "qec", "--circuit", "-d", str(q["distance"]), "--rounds", str(q["rounds"]), "--p", repr(q["p"])], capture_output=True, text=True).stdout
|
||
rep.add(q["id"], "wai.qec", "the engine's own generator writes the same text, byte for byte", mine == text, True, mine == text)
|
||
for k, v in hand.items():
|
||
if a.tier == "ci" and k not in CI_HAND:
|
||
continue
|
||
print("·", k, file=sys.stderr, flush=True)
|
||
check_qec_entry({"id": k, "tier": a.tier}, v["text"], runner, rep, qplan)
|
||
judge_qec(qplan, rep, FAMILY_WISE_ALPHA / 3)
|
||
entries_run = {e["id"] for e in es} | {q["id"] for q in qs} | {k for k in hand if a.tier == "full" or k in CI_HAND}
|
||
apply_known(rep, entries_run)
|
||
check_expected(rep, entries_run)
|
||
vers = json.loads(subprocess.run([sys.executable, "-I", os.path.join(HERE, "engines.py"), "--versions"], capture_output=True, text=True).stdout)
|
||
statuses = {}
|
||
for c in rep.checks:
|
||
statuses[c["status"]] = statuses.get(c["status"], 0) + 1
|
||
out = {
|
||
"about": "One run of the cross-simulator corpus: every check, its measured value and its tolerance. tools/compare.py writes it.",
|
||
"tier": a.tier,
|
||
"inject": a.inject,
|
||
"host": host(),
|
||
"versions": vers,
|
||
"engine_under_test": {"driver": os.path.basename(os.environ["WAI_CROSS_SIMULATOR"]), "build": os.environ.get("WAI_BUILD_INFO", "")},
|
||
"seconds": round(time.time() - started, 1),
|
||
"summary": statuses,
|
||
"checks": rep.checks,
|
||
}
|
||
if a.out:
|
||
with open(a.out, "w") as f:
|
||
json.dump(out, f, indent=1, sort_keys=False, default=str)
|
||
f.write("\n")
|
||
fails = [c for c in rep.checks if c["status"] == "fail"]
|
||
for c in rep.checks:
|
||
if c["status"] in ("fail", "refused", "not run", "known"):
|
||
print("%-8s %-34s %-34s %s | measured %s, tolerance %s%s" % (c["status"].upper(), c["entry"][:34], c["engine"][:34], c["quantity"][:110], fmt(c["measured"]), fmt(c["tolerance"]), (" — " + c["note"][:240]) if c.get("note") else ""))
|
||
print("summary: %s; %.0f s" % (", ".join("%s %d" % kv for kv in sorted(statuses.items())), time.time() - started))
|
||
print("FAIL" if fails else "PASS")
|
||
if not a.keep:
|
||
shutil.rmtree(work, ignore_errors=True)
|
||
sys.exit(1 if fails else 0)
|
||
|
||
|
||
def fmt(x):
|
||
if isinstance(x, float):
|
||
return "%.3g" % x
|
||
return str(x)[:80]
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|