df011db75a
compare.py is the corpus branch's, with this branch's changes made again on it: the bit-exact method held to OpenQASM's state with the global phase its result states; error-correction circuits with C_XYZ, E and PAULI_CHANNEL_1 compared, a model the other engine cannot give held by its sampler; a wide entry that names its engines and bond. The expected refusals follow the reader: the colour code, E and the Pauli channel are no longer refused, and a channel with no independent form must be. A timed row runs the bit-exact method asked to hold a gate whole. Co-Authored-By: Claude Opus 5.5 <[email protected]>
214 lines
13 KiB
Python
214 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Run time and peak memory of each engine on the same circuits, on this host.
|
|
|
|
perf.py [--out PERF.json] [--repeat N] [--quick]
|
|
|
|
Each run is a process of its own. Its time is the engine's own call (`seconds`, from the adapter
|
|
or the driver; TIMERS says where each engine's starts and stops). Its memory is the peak resident
|
|
set of the process that ran the engine, as the operating system counts it for that process
|
|
alone (engines.py): the driver's for the engine under test, and the adapter's for another engine,
|
|
which runs in the adapter with its interpreter and packages. `baseline` is the same measure given
|
|
the smallest job, so the difference is about what the circuit took.
|
|
|
|
THREADS says what each engine was given. The engine under test has no thread setting in its
|
|
public interface: its state vector and tensor network use every CPU the process may run on, and
|
|
its other methods run on one thread. On Linux its one-thread rows are made by pinning the process
|
|
to one CPU (taskset), and on macOS there are none.
|
|
|
|
A shared host's other work moves these numbers, by several times at worst: the load average
|
|
before and after each case is recorded beside it, and each case is run `repeat` times, with the
|
|
minimum and the median kept. Read them as this host on this day, not as a property of an engine.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
import compare # noqa: E402
|
|
import corpus # noqa: E402
|
|
|
|
# What each engine was given: "all" is every CPU the process may run on (std::thread::
|
|
# available_parallelism), "one" a method with no threads of its own, "given" the count passed in
|
|
# its own setting, and "not set" an engine the adapter gives no thread setting at all.
|
|
THREADS = {
|
|
"wai.statevector": "all", "wai.tensor_network": "all", "wai.exact": "one", "wai.exact_whole": "one", "wai.stabilizer": "one", "wai.qec": "one", "wai.mps": "one",
|
|
"aer.statevector": "given", "aer.stabilizer": "given", "aer.mps": "given", "qsim.statevector": "given",
|
|
"ddsim.statevector": "not set", "stim.tableau": "not set", "quimb.dense": "not set", "quimb.mps": "not set", "quimb.tensor_network": "not set",
|
|
}
|
|
# Where each engine's `seconds` starts and stops.
|
|
TIMERS = {
|
|
"wai": "the engine's run, after the program is read and lowered (the driver's own clock)",
|
|
"aer": "the simulator's own time for the experiment, as its result states it (time_taken); the conversion of the circuit to its own form is outside it",
|
|
"qsim": "the simulate call, which converts the circuit to its own form inside it",
|
|
"ddsim": "the run call, which converts the circuit to its own form inside it",
|
|
"stim": "the tableau's or sampler's run, after the circuit text is parsed",
|
|
"quimb": "the building of the circuit object, which applies each gate as it is added, and the read-out",
|
|
}
|
|
|
|
|
|
def cases(quick):
|
|
out = []
|
|
widths = (20, 24) if quick else (20, 24, 26, 28)
|
|
for n in widths:
|
|
steps = corpus.basis_prep(n, (1 << n) // 3) + corpus.qft_steps(n)
|
|
engines = ["wai.statevector", "aer.statevector", "qsim.statevector"]
|
|
if n <= 24:
|
|
engines += ["ddsim.statevector", "wai.exact"]
|
|
if n <= 20:
|
|
engines.append("quimb.dense")
|
|
out.append({"id": "qft_n%d" % n, "what": "the QFT of a basis state, the whole state", "kind": "state", "n": n, "steps": steps, "engines": engines})
|
|
for n in ((20, 24) if quick else (20, 24, 26)):
|
|
steps = corpus.random_rotations(n, 20, 100 + n)
|
|
# No decision-diagram engine here: a circuit with no structure to share is outside what
|
|
# the method is for (the conformance results record its time on the mirror circuits).
|
|
engines = ["wai.statevector", "aer.statevector", "qsim.statevector"]
|
|
if n <= 20:
|
|
engines.append("quimb.dense")
|
|
out.append({"id": "rotations_n%d" % n, "what": "20 layers of arbitrary U gates on a brickwork of cx, the whole state", "kind": "state", "n": n, "steps": steps, "engines": engines})
|
|
# Past 16 the gates under n - 1 controls stand whole, or (at 17) past what a program may be
|
|
# in standard gates: a width only the state vectors that hold them whole reach.
|
|
for n in (10, 14, 16, 20):
|
|
# The bit-exact method runs standard gates unless asked (its bits are theirs): to 14
|
|
# qubits as it runs by default, and at every width asked to hold the gate whole.
|
|
engines = ["wai.statevector"] + (["wai.exact"] if n <= 14 else []) + ["wai.exact_whole", "aer.statevector", "qsim.statevector", "ddsim.statevector"]
|
|
out.append({"id": "grover_n%d" % n, "what": "two Grover iterations on %d qubits: four Z gates under %d controls, the whole state" % (n, n - 1), "kind": "state", "n": n, "steps": corpus.grover_steps(n, (1 << n) // 3, 2), "engines": engines})
|
|
for n, depth in ((1000, 30), (5000, 10)):
|
|
steps = corpus.random_clifford(n, depth, 200 + n)
|
|
engines = ["wai.stabilizer", "stim.tableau"] + (["aer.stabilizer"] if n <= 1000 else [])
|
|
out.append({"id": "clifford_n%d" % n, "what": "random Clifford layers: the tableau, and its stabilizer group read out", "kind": "stabilizers", "n": n, "steps": steps, "engines": engines})
|
|
for d in (5, 11) if not quick else (5,):
|
|
out.append({"id": "surface_z_d%d_sample" % d, "what": "the rotated surface-code memory experiment, d = rounds = %d, p = 0.001: 10^6 shots of every detector" % d, "kind": "qec_sample", "qec": {"code": "surface_code:rotated_memory_z", "distance": d, "rounds": d, "p": 0.001}, "shots": 1000000, "engines": ["wai.qec", "stim.tableau"]})
|
|
out.append({"id": "surface_z_d%d_model" % d, "what": "the same circuit's detector error model", "kind": "qec_model", "qec": {"code": "surface_code:rotated_memory_z", "distance": d, "rounds": d, "p": 0.001}, "engines": ["wai.qec", "stim.tableau"]})
|
|
n = 40
|
|
x = 366503875925
|
|
steps = corpus.basis_prep(n, x) + corpus.qft_steps(n)
|
|
out.append({"id": "qft_n40_amplitudes", "what": "six amplitudes of the 40-qubit QFT of a basis state", "kind": "amplitudes", "n": n, "steps": steps, "bitstrings": [format(y, "040b") for y in (1, 12345, 2**39 + 7, 999999999, 2**40 - 1, 31337)], "engines": ["wai.mps", "aer.mps", "quimb.mps", "wai.tensor_network", "quimb.tensor_network"]})
|
|
return out
|
|
|
|
|
|
def run_once(python, engine, job, work, pin_one):
|
|
jp = os.path.join(work, "job.json")
|
|
ap = os.path.join(work, "answer.json")
|
|
if job["kind"] == "state":
|
|
job = dict(job, state_out=os.path.join(work, "state.bin"))
|
|
with open(jp, "w") as f:
|
|
json.dump(job, f)
|
|
cmd = [python, "-I", os.path.join(HERE, "engines.py"), engine, jp, ap]
|
|
if pin_one:
|
|
cmd = ["taskset", "-c", "0"] + cmd
|
|
t0 = time.perf_counter()
|
|
p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
_, status, ru = os.wait4(p.pid, 0)
|
|
wall = time.perf_counter() - t0
|
|
err = p.stderr.read().decode(errors="replace")
|
|
p.stderr.close()
|
|
try:
|
|
ans = json.load(open(ap))
|
|
except Exception: # noqa: BLE001
|
|
ans = {"ok": False, "error": "no answer: %s" % err[-500:]}
|
|
# ru_maxrss: bytes on macOS, KiB on Linux. The driver's own peak is a child of the adapter's,
|
|
# and wait4 reports the largest of the process and its waited-for children.
|
|
# The engine's own process, as engines.py measured it. Not this process's wait4: on Linux a
|
|
# child's ru_maxrss starts at what its parent held when it forked, which is this harness.
|
|
rss = ans.get("engine_peak_rss_bytes") or (ru.ru_maxrss if platform.system() == "Darwin" else None)
|
|
for f in os.listdir(work):
|
|
if f.startswith("state.bin"):
|
|
os.unlink(os.path.join(work, f))
|
|
return ans, wall, rss
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--out")
|
|
ap.add_argument("--repeat", type=int, default=5)
|
|
ap.add_argument("--quick", action="store_true")
|
|
ap.add_argument("--only", help="run the cases whose id starts with this")
|
|
a = ap.parse_args()
|
|
python = sys.executable
|
|
work = tempfile.mkdtemp(prefix="qcorpus-perf-")
|
|
threads = os.cpu_count()
|
|
linux = platform.system() == "Linux" and shutil.which("taskset")
|
|
rows = []
|
|
base = {}
|
|
|
|
def baseline(eng, kind):
|
|
"""The same process given the smallest job of the kind: what the interpreter and the
|
|
engine's packages take before any circuit."""
|
|
if (eng, kind) not in base:
|
|
tiny = {"kind": kind, "perf": True, "n": 2, "steps": [corpus.g("h", 0), corpus.g("cx", 0, 1)], "paulis": ["XX"], "bitstrings": ["00"], "text": "R 0\nM 0\nDETECTOR rec[-1]\n", "shots": 64}
|
|
ans, _, rss = run_once(python, eng, tiny, work, False)
|
|
base[(eng, kind)] = rss if ans.get("ok") and rss else None
|
|
return base[(eng, kind)]
|
|
|
|
for c in cases(a.quick):
|
|
job = {k: v for k, v in c.items() if k not in ("engines", "what", "qec")}
|
|
job["perf"] = True
|
|
if c.get("qec"):
|
|
job["text"] = open(os.path.join(compare.ROOT, "corpus", "qec", "surface_z_d%d.circuit" % c["qec"]["distance"])).read()
|
|
if c["kind"] == "stabilizers":
|
|
# The engine under test is handed the group to check; it is the tableau's run that is timed.
|
|
import exact
|
|
|
|
tb = exact.Tableau(c["n"])
|
|
for s in c["steps"]:
|
|
tb.gate(s["g"], s["q"])
|
|
rows_ = exact.reduced_group(tb.stabilizers(), c["n"])[0]
|
|
job["paulis"] = [exact.pauli_text(r, c["n"])[1:] for r in rows_[:1]]
|
|
if a.only and not c["id"].startswith(a.only):
|
|
continue
|
|
modes = [False] + ([True] if linux and c["kind"] in ("state", "stabilizers", "qec_sample") and c.get("n", 0) <= 24 else [])
|
|
for pin in modes:
|
|
for eng in c["engines"]:
|
|
j = dict(job, threads=1 if pin else threads)
|
|
runs = []
|
|
load_before = os.getloadavg()
|
|
for _ in range(a.repeat):
|
|
ans, wall, rss = run_once(python, eng, j, work, pin)
|
|
if not ans.get("ok"):
|
|
runs = None
|
|
why = ans.get("error") or ans.get("unsupported") or ans.get("refused")
|
|
break
|
|
runs.append((ans.get("seconds"), wall, rss, ans))
|
|
load_after = os.getloadavg()
|
|
given = THREADS.get(eng, "not set")
|
|
row = {"case": c["id"], "what": c["what"], "engine": eng, "cpus": 1 if pin else threads, "threads": "1 (pinned)" if pin else {"all": "all (%d)" % threads, "one": "1 (no threads)", "given": str(threads), "not set": "not set"}[given], "timer": TIMERS[eng.split(".")[0]], "load_before": [round(x, 1) for x in load_before], "load_after": [round(x, 1) for x in load_after]}
|
|
if runs is None:
|
|
row["not_run"] = str(why)[:300]
|
|
else:
|
|
secs = [r[0] for r in runs if r[0] is not None]
|
|
last = runs[-1][3]
|
|
rss = [r[2] for r in runs if r[2]]
|
|
row.update({
|
|
"seconds_min": min(secs) if secs else None,
|
|
"seconds_median": statistics.median(secs) if secs else None,
|
|
"wall_median": statistics.median(r[1] for r in runs),
|
|
"peak_rss_mib": round(max(rss) / 2**20, 1) if rss else None,
|
|
"baseline_rss_mib": (lambda b: round(b / 2**20, 1) if b else None)(baseline(eng, c["kind"])),
|
|
"repeats": len(runs),
|
|
# The operations the engine ran: the driver's count of its ops, or the gates
|
|
# of Aer's circuit after it was converted.
|
|
"ops": last.get("n_ops") or (last.get("settings") or {}).get("gates"),
|
|
"settings": {k: v for k, v in (last.get("settings") or {}).items() if k in ("fusion", "fusion_threshold", "fusion_max_qubit", "precision", "arithmetic", "cutoff", "cutoff_mode")} or None,
|
|
})
|
|
if last.get("call_seconds") is not None:
|
|
row["call_seconds_median"] = statistics.median(r[3].get("call_seconds") for r in runs)
|
|
rows.append(row)
|
|
print(json.dumps(row), flush=True)
|
|
out = {"about": "Run time and peak memory per engine on one host: tools/perf.py. Times are each engine's own call (timers says where each starts and stops), the minimum and median of repeated runs; memory is the peak resident set of the process that ran the engine.", "timers": TIMERS, "host": compare.host(), "versions": json.loads(subprocess.run([python, "-I", os.path.join(HERE, "engines.py"), "--versions"], capture_output=True, text=True).stdout), "engine_under_test": os.environ.get("WAI_BUILD_INFO", ""), "repeat": a.repeat, "rows": rows}
|
|
if a.out:
|
|
with open(a.out, "w") as f:
|
|
json.dump(out, f, indent=1)
|
|
f.write("\n")
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|