open-standards/wai/quantum-cross-simulator/provider/test.sh

#!/usr/bin/env bash
# provider/test.sh: the provider template through the command line, as a person uses it.
#
#   bash provider/test.sh path/to/wai-quantum path/to/python
#
# It adds the template as a `process` provider, holds it to the protocol's conformance vectors
# (`providers test`), runs a program on it with `run --provider`, and verifies the receipt the
# command line seals. Everything is in a directory of its own, removed at the end: no key, no
# registry and no credential of the person's is touched.
set -euo pipefail
W="$1"
PY="$2"
case "$W" in */*) W="$(cd "$(dirname "$W")" && pwd)/$(basename "$W")" ;; esac
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
export XDG_CONFIG_HOME="$tmp/config" WAI_QUANTUM_PROVIDERS="$tmp/providers.json" WAI_QUANTUM_CREDENTIALS=file
unset WAI_QUANTUM_KEY
fail() { echo "provider: $*" >&2; exit 1; }

"$W" providers add sim --kind process -- "$PY" -I "$HERE/seeded_statevector_provider.py" > "$tmp/add.txt"
"$W" providers describe sim > "$tmp/describe.txt" || fail "describe failed: $(cat "$tmp/describe.txt")"
if ! "$W" providers test sim > "$tmp/test.txt" 2>&1; then
  cat "$tmp/test.txt" >&2
  fail "the protocol's conformance vectors failed"
fi
grep -qE '^ *[1-9][0-9]* passed, 0 failed' "$tmp/test.txt" || { cat "$tmp/test.txt" >&2; fail "a vector failed, or none ran"; }
echo "provider: conformance vectors passed"
sed 's/^/  /' "$tmp/test.txt" | head -40

cat > "$tmp/ghz.qasm" <<'EOF'
OPENQASM 3.0;
include "stdgates.inc";
qubit[3] q;
bit[3] c;
h q[0];
cx q[0], q[1];
cx q[1], q[2];
t q[2];
c = measure q;
EOF
"$W" run "$tmp/ghz.qasm" --provider sim --shots 4000 --json > "$tmp/job.json" || fail "run --provider failed: $(head -c 2000 "$tmp/job.json")"
"$W" verify "$tmp/job.json" > "$tmp/verify.txt" || fail "the receipt did not verify: $(cat "$tmp/verify.txt")"
grep -q VALID "$tmp/verify.txt" || fail "verify did not say VALID"
"$PY" -I - "$tmp/job.json" <<'EOF'
import json, sys
j = json.load(open(sys.argv[1]))
r = j["result"]
counts = r["counts"]
assert set(counts) <= {"000", "111"}, counts
assert sum(counts.values()) == 4000, counts
assert r["accuracy"]["class"] == "simulated", r["accuracy"]
assert r["provider"]["provenance"].startswith("simulator"), r["provider"]
ideal = r.get("ideal") or {}
tvd = ideal.get("tvd", ideal.get("total_variation"))
print("provider: run --provider sealed %s; counts %s; ideal %s" % (r["method"], counts, json.dumps(ideal)[:200]))
EOF
# The same job twice gives the same counts: the device is seeded.
"$W" run "$tmp/ghz.qasm" --provider sim --shots 4000 --json > "$tmp/job2.json"
"$PY" -I -c 'import json,sys; a,b=(json.load(open(p))["result"]["counts"] for p in sys.argv[1:]); assert a == b, (a, b); print("provider: seeded, the same counts twice")' "$tmp/job.json" "$tmp/job2.json"
echo "provider: PASS"