open-standards/wai/quickstart/test/archive_doctor.py

#!/usr/bin/env python3
"""archive_doctor.py: `wai doctor` run from an unpacked quickstart archive, and held to what the
archive itself decides.

    archive_doctor.py <unpacked-archive-dir> [--timeout S]
    archive_doctor.py --wai <wai> --require <check-id> [--require <check-id>]... [--timeout S]

The first form runs <dir>/bin/wai (wai.exe on Windows) as `wai doctor --json`, with <dir>/bin first
on PATH, as an installed archive is, and requires every `install.*` and `stack.*` check to be `pass`:
this wai and what a prompt reaches, the relay and the room service beside it and of its version, and
the room client's and Studio's files. These six must be in the report:
install.version, install.path, stack.wai-relay, stack.wai-room, stack.web, stack.studio; and what
install.version and each of the four stack checks found must be in the archive, since `wai dev`
would otherwise take it from elsewhere on PATH or from the environment (WAI_RELAY_BIN, WAI_WEB, ...),
which would hide a piece the archive lacks. The other
checks (the ports, the browsers, the keystore, the keychain) depend on the machine it runs on: they
are printed and not required.

Before the doctor, the first form starts the two programs the doctor asks for their version,
bin/wai-relay and bin/wai-room, once each as the doctor asks them (`--version --json`), and prints
how long each took. The first start of a program written moments ago waits on the system's scan of
a new executable, and that wait would count against the doctor's 6 s for its whole run, a bound
for a person at a prompt and not one of the archive. On the Windows runner (Windows's
real-time protection on) the first start of a fresh copy of the 0.6.0 release's wai-relay.exe took
0.4 to 0.7 s, its second 11 to 16 ms; the rehearsal's larger debug programs, started by the doctor
on a host its build had just loaded, did not finish within the 6 s, and the doctor reported
stack.wai-relay and stack.wai-room not checked, while every group of checks that starts no program
finished. A first start that does not end within the timeout fails the check: that is a program
that hangs, not a scan.

The second form runs a `wai` as it is, in the environment as it is, and requires the checks named.

Every check is printed, one to a line, with its state. Exit 0 when every required check passed; 1
when one did not, is not in the report, or there is no report (wai did not run, did not answer in
time, or printed something else), or when a first start did not end in time; 2 for a usage error
or a directory with no bin/wai. --timeout bounds the doctor's run and each first start (120 s).

Standard library only; runs on Python 3.9.
"""
from __future__ import annotations

import json
import os
import subprocess
import sys
import time

ARCHIVE_REQUIRED = ["install.version", "install.path", "stack.wai-relay", "stack.wai-room", "stack.web", "stack.studio"]
ARCHIVE_PREFIXES = ("install.", "stack.")
# The checks whose `facts.path` must be inside the archive, and under which of its directories.
ARCHIVE_INSIDE = {"install.version": "bin", "stack.wai-relay": "bin", "stack.wai-room": "bin", "stack.web": "share", "stack.studio": "share"}
# The programs the doctor asks for their version, started once before it (the module's text).
FIRST_STARTS = ["wai-relay", "wai-room"]


def inside(path, root):
    """Whether `path` is `root` or under it, both resolved."""
    p, r = os.path.normcase(os.path.realpath(path)), os.path.normcase(os.path.realpath(root))
    return p == r or p.startswith(r.rstrip(os.sep) + os.sep)


def usage(msg):
    print(f"archive_doctor.py: {msg}", file=sys.stderr)
    print(__doc__.split("\n\n")[1], file=sys.stderr)
    sys.exit(2)


def first_start(exe, env, timeout):
    """`exe --version --json` started once, as the doctor asks it: how long it took to end, or why it
    did not. Its exit status is the doctor's to judge."""
    began = time.monotonic()
    try:
        subprocess.run([exe, "--version", "--json"], env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
                       stderr=subprocess.DEVNULL, timeout=timeout)
    except subprocess.TimeoutExpired:
        return None, f"did not end within {timeout:g} s"
    except OSError as e:
        return None, f"could not be started: {e}"
    return time.monotonic() - began, None


def report(wai, env, timeout):
    """The envelope `wai doctor --json` prints, or why there is none."""
    try:
        p = subprocess.run([wai, "doctor", "--json"], env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE, timeout=timeout)
    except subprocess.TimeoutExpired:
        return None, f"{wai} doctor --json did not end within {timeout} s"
    except OSError as e:
        return None, f"{wai} could not be started: {e}"
    try:
        v = json.loads(p.stdout)
    except ValueError:
        err = p.stderr.decode("utf-8", "replace").strip().splitlines()[-1:] or [""]
        return None, f"{wai} doctor --json exited {p.returncode} and printed no envelope: {err[0]}"
    if not isinstance(v, dict) or v.get("wai") != "envelope/1" or v.get("op") != "toolchain.doctor":
        return None, f"{wai} doctor --json printed another document: {str(v)[:200]}"
    checks = v.get("data", {}).get("checks") if isinstance(v.get("data"), dict) else None
    if not isinstance(checks, list):
        return None, f"{wai} doctor --json printed an envelope with no checks: {str(v)[:200]}"
    return v, None


def main(argv):
    timeout = 120
    wai = None
    required = []
    rest = []
    i = 0
    while i < len(argv):
        a = argv[i]
        if a in ("--timeout", "--wai", "--require"):
            if i + 1 >= len(argv):
                usage(f"{a} takes a value")
            val = argv[i + 1]
            if a == "--timeout":
                try:
                    timeout = float(val)
                except ValueError:
                    usage(f"--timeout {val}: a number of seconds")
            elif a == "--wai":
                wai = val
            else:
                required.append(val)
            i += 2
            continue
        if a.startswith("--"):
            usage(f"{a}: not an option")
        rest.append(a)
        i += 1

    env = dict(os.environ)
    if wai is None:
        if len(rest) != 1 or required:
            usage("name one unpacked archive, or --wai with --require")
        root = rest[0]
        exe = "wai.exe" if os.name == "nt" else "wai"
        bin_dir = os.path.abspath(os.path.join(root, "bin"))
        wai = os.path.join(bin_dir, exe)
        if not os.path.isfile(wai):
            usage(f"{root}: no bin/{exe} in it")
        env["PATH"] = bin_dir + os.pathsep + env.get("PATH", "")
        required = list(ARCHIVE_REQUIRED)
        prefixes = ARCHIVE_PREFIXES
        within = {cid: os.path.join(root, sub) for cid, sub in ARCHIVE_INSIDE.items()}
        print(f"wai doctor from the archive {os.path.abspath(root)}, its bin first on PATH")
        exe_ext = ".exe" if os.name == "nt" else ""
        for name in FIRST_STARTS:
            exe = os.path.join(bin_dir, name + exe_ext)
            if not os.path.isfile(exe):
                continue  # the doctor's stack check says so
            secs, why = first_start(exe, env, timeout)
            if why is not None:
                print(f"FAIL  the first start of bin/{name}{exe_ext} --version --json {why}")
                return 1
            print(f"first start of bin/{name}{exe_ext}: {secs:.1f} s, before the doctor's own run")
    else:
        if rest or not required:
            usage("--wai takes --require, and no archive")
        prefixes = ()
        within = {}
        print(f"wai doctor from {wai}")

    v, why = report(wai, env, timeout)
    if v is None:
        print(f"FAIL  {why}")
        return 1
    checks = v["data"]["checks"]
    states, paths = {}, {}
    for c in checks:
        cid, state = str(c.get("id")), str(c.get("state"))
        states[cid] = state
        facts = c.get("facts") if isinstance(c.get("facts"), dict) else {}
        paths[cid] = facts.get("path")
        need = cid in required or cid.startswith(prefixes)
        print(f"{'required' if need else 'printed ':<9} {state:<15} {cid:<22} {c.get('message', '')}")
    bad = []
    for cid in required:
        if cid not in states:
            bad.append(f"{cid} is not in the report")
    for cid, state in states.items():
        if (cid in required or cid.startswith(prefixes)) and state != "pass":
            bad.append(f"{cid} is {state}")
        elif cid in within and not (isinstance(paths[cid], str) and inside(paths[cid], within[cid])):
            bad.append(f"{cid} passed with {paths[cid]}, which is not in the archive's {os.path.basename(within[cid])}/")
    if bad:
        for b in bad:
            print(f"FAIL  {b}")
        return 1
    print(f"ok    every required check passed ({len([c for c in states if c in required or c.startswith(prefixes)])})")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))