open-standards/wai/quantum-conformance/landscape/reproduce.sh

#!/bin/sh
# Check the stated result (stated-result.md), from a clean checkout. Needs Rust through rustup, which
# reads rust-toolchain.toml. Node, wasm-bindgen 0.2.127 and the wasm32-unknown-unknown target are needed
# only for the last, optional check.
#   quantum-conformance/landscape/reproduce.sh            each stated value, and what this machine got
#   quantum-conformance/landscape/reproduce.sh --print    this machine's values only, to state them
set -eu
HERE=$(cd "$(dirname "$0")" && pwd)
WAI=$(cd "$HERE/../.." && pwd)
PRINT=0; [ "${1:-}" = "--print" ] && PRINT=1
cd "$WAI/wai-quantum-cli"
cargo build --release --locked >/dev/null 2>&1 || cargo build --release --locked
T=$(cargo metadata --format-version 1 --no-deps | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p')
B=$T/release/wai-quantum
TMP=$(mktemp -d)
sha() { if command -v sha256sum >/dev/null; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; }
fail=0
state() {
  if [ $PRINT = 1 ]; then echo "$1: $2"; return; fi
  if [ "$2" = "$3" ]; then echo "PASS  $1: $2"; else echo "FAIL  $1: this machine got $2, the stated value is $3"; fail=1; fi
}

# 1. The twelve talks, a thousand seeds.
"$B" landscape replay --example talks --seed 1 --seeds 1000 > "$TMP/talks.txt"
"$B" landscape run --example talks --seed 1 --json --demo-key > "$TMP/talks1.json"
state "talks: arrangements checked" "$(sed -n 's/.*"arrangements":\([0-9]*\).*/\1/p' "$TMP/talks1.json")" "531441"
state "talks: lowest energy" "$(sed -n 's/.*"lowest":\(-*[0-9]*\).*/\1/p' "$TMP/talks1.json")" "-10"
state "talks: arrangements at the lowest energy" "$(sed -n 's/.*"at_lowest":\([0-9]*\).*/\1/p' "$TMP/talks1.json")" "2"
state "talks: runs of 1000 that reach it" "$(awk '$4 == "-10"' "$TMP/talks.txt" | wc -l | tr -d ' ')" "1000"
state "talks: accumulates a run" "$(awk '{print $6}' "$TMP/talks.txt" | sort -u | tr '\n' ' ' | sed 's/ $//')" "13440"
state "talks: replay digest at seed 1" "$(sed -n 's/.*"replay":"\([0-9a-f]*\)".*/\1/p' "$TMP/talks1.json")" "6c40700b29e6193abe0a54d8424f2c65f6658a55849176b21f2e2cce4bca05fc"
state "talks: the lines of seeds 1 to 1000, SHA-256" "$(sha "$TMP/talks.txt")" "9ad4367cf31a607c3edfc3732d740824e4b7d56ae141c5bf772fd087347209dd"

# 2. Every worked example, at its own temperature, at 0 and at 0.5, two hundred seeds each.
: > "$TMP/examples.txt"
for e in $("$B" landscape examples | awk '{print $1}'); do
  for t in "" 0 0.5; do
    echo "== $e ${t:-own}" >> "$TMP/examples.txt"
    if [ -n "$t" ]; then "$B" landscape replay --example "$e" --seed 0 --seeds 200 --temperature "$t" >> "$TMP/examples.txt" 2>&1 || true
    else "$B" landscape replay --example "$e" --seed 0 --seeds 200 >> "$TMP/examples.txt" 2>&1 || true; fi
  done
done
state "examples: runs that solved, of those that ran" "$(grep -c ' solved ' "$TMP/examples.txt") of $(awk 'NF == 7' "$TMP/examples.txt" | wc -l | tr -d ' ')" "3637 of 7800"
state "examples: every line, SHA-256" "$(sha "$TMP/examples.txt")" "d9db5db3fc24f35a8d8c389e720ccbe36e7c3c26bcb16672db064cddc892e95b"
state "examples: the digest of every answer at 20 seeds" "$("$B" landscape corpus --seeds 20)" "a84fb1e13592511ce7f9ebd2dde1af711bcc16786fc2f08d79032cab2e93158a"

# 3. The scale table, which is this directory's.
mkdir -p "$TMP/table"
"$B" landscape table --out "$TMP/table" > /dev/null
state "table: table.json, SHA-256" "$(sha "$TMP/table/table.json")" "aea3a97c927fbce24197d0065f63e1f36a2bda6c359fa67f956c4ab752c32dd6"
state "table: table.json the same as this directory's" "$(cmp -s "$TMP/table/table.json" "$HERE/table.json" && echo same || echo different)" "same"
state "table: table.md the same as this directory's" "$(cmp -s "$TMP/table/table.md" "$HERE/table.md" && echo same || echo different)" "same"
tally() { sed -n "s/.*$1 reached the target more often than every classical solver in \([0-9]*\), tied the best of them in \([0-9]*\), and was behind the best of them in \([0-9]*\)\..*/\1 \2 \3/p" "$TMP/table/table.md"; }
state "table: at the default start, ahead, tied and behind the best classical method" "$(tally 'at the default start')" "2 45 21"
state "table: at the hot start, ahead, tied and behind the best classical method" "$(tally 'at the hot start, the largest row of couplings')" "0 38 30"

# 4. The conformance set and the trial, which are this directory's.
"$B" landscape vectors > "$TMP/vectors.json"
state "vectors: the same as this directory's" "$(cmp -s "$TMP/vectors.json" "$HERE/vectors.json" && echo same || echo different)" "same"
"$B" landscape trial-assignment > "$TMP/trial-assignment.json"
state "trial: the same as this directory's" "$(cmp -s "$TMP/trial-assignment.json" "$HERE/trial-assignment.json" && echo same || echo different)" "same"

# 5. Optional: the same answers from the browser build, if Node and its tools are here. The
#    comparison's own exit status decides: POSIX sh has no pipefail, so nothing is piped.
if command -v node >/dev/null && command -v wasm-bindgen >/dev/null && rustup target list --installed 2>/dev/null | grep -q wasm32-unknown-unknown; then
  WB=$(wasm-bindgen --version | awk '{print $2}')
  if [ "$WB" != 0.2.127 ]; then
    echo "SKIP  native against WebAssembly: wasm-bindgen $WB is here, and wai-web's lockfile takes 0.2.127"
  elif ! (cd "$WAI/wai-web" && cargo build --locked --release --target wasm32-unknown-unknown --features quantum_engine >/dev/null 2>&1); then
    echo "FAIL  native against WebAssembly: the browser build did not build"
    fail=1
  else
    W=$(cd "$WAI/wai-web" && cargo metadata --format-version 1 --no-deps | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p')
    wasm-bindgen --target nodejs --out-dir "$TMP/pkg" --out-name wai_web "$W/wasm32-unknown-unknown/release/wai_web.wasm"
    cp "$WAI/wai-web/demo/verify_landscape_wasm.mjs" "$TMP/verify.mjs"
    if out=$(node "$TMP/verify.mjs" "$TMP/pkg/wai_web.js" "$B" 200 2>&1); then
      echo "PASS  $(printf '%s\n' "$out" | tail -1)"
    else
      echo "FAIL  $(printf '%s\n' "$out" | tail -1)"
      fail=1
    fi
  fi
else
  echo "SKIP  native against WebAssembly: needs Node, wasm-bindgen and the wasm32-unknown-unknown target"
fi

[ $PRINT = 1 ] && exit 0
[ $fail = 0 ] && echo "Every stated value reproduced on this machine." || echo "At least one stated value did not reproduce."
exit $fail