open-standards/scripts/check-advisories-selftest.sh

#!/usr/bin/env bash
#
# check-advisories-selftest.sh — check-advisories.py can fail.
#
# A check that cannot fail is decoration. This feeds the check pre-captured
# `cargo audit --json` documents through `--from-json` and asserts that it
#   1. passes when every advisory found is allowed and every allowance is used;
#   2. fails when an advisory is reported that the allow list does not name
#      (the case this check exists for);
#   3. fails when an allow-list entry matches nothing, so an excuse cannot
#      outlive the exposure it was written for;
#   4. fails when a version-pinned allowance is offered a version it does not
#      name — the whole point of pinning it;
#   5. fails when a version-pinned allowance names a pair that is gone, so the
#      pin cannot keep describing a tree that moved;
#   6. fails, and says it could not audit, when the document does not parse:
#      an error is never read as "clean".
#
# The documents are generated from the check's own ALLOW table, imported rather
# than scraped, so the two cannot drift apart. Nothing here runs cargo or reads
# the advisory database, so it needs no network and no lockfile.
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
check="$here/check-advisories.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT

# gen <out> [mutation]  — a cargo-audit document built from the real ALLOW.
#   (none)      every allowance, each at a version it covers
#   extra       plus an advisory nobody allows
#   drop-one    one allowance missing, so it matches nothing
#   wrong-ver   a version-pinned allowance at a version it does not name
#   drop-pinned a version-pinned allowance's named pair removed
gen() {
  python3 -I - "$check" "$1" "${2:-none}" <<'PY'
import importlib.util, json, sys
path, out, mode = sys.argv[1], sys.argv[2], sys.argv[3]
spec = importlib.util.spec_from_file_location("chk", path)
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)           # __name__ != "__main__", so main() stays put

rows = []
for adv, entry in m.ALLOW.items():
    if isinstance(entry, str):
        rows.append((adv, "c", "1.0.0"))
    else:
        for pair in entry["only"]:
            crate, ver = pair.rsplit(" ", 1)
            rows.append((adv, crate, ver))

pinned = [a for a, e in m.ALLOW.items() if not isinstance(e, str)]
if mode == "extra":
    rows.append(("RUSTSEC-2099-0001", "c", "1.0.0"))
elif mode == "drop-one":
    rows.pop(0)
elif mode == "wrong-ver":
    assert pinned, "no version-pinned allowance to test"
    rows = [(a, c, ("9.9.9" if a == pinned[0] else v)) for a, c, v in rows]
elif mode == "drop-pinned":
    assert pinned, "no version-pinned allowance to test"
    rows = [(a, c, v) for a, c, v in rows if a != pinned[0]]
    rows.append((pinned[0], "c", "1.0.0"))   # present, but not at a named pair

doc = {"vulnerabilities": {"found": bool(rows), "count": len(rows), "list": [
    {"advisory": {"id": a, "title": "t"}, "package": {"name": c, "version": v}}
    for a, c, v in rows]}, "warnings": {}}
open(out, "w").write(json.dumps(doc))
PY
}

fails=0
expect() { # expect <pass|fail> <what> <needle> <file>
  local want=$1 what=$2 needle=$3 file=$4 out rc
  out=$(python3 -I "$check" --from-json "$file" 2>&1); rc=$?
  if [ "$want" = pass ] && [ $rc -ne 0 ]; then
    echo "FAIL  $what: the check failed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return
  fi
  if [ "$want" = fail ] && [ $rc -eq 0 ]; then
    echo "FAIL  $what: the check passed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return
  fi
  if [ -n "$needle" ] && ! grep -q -- "$needle" <<<"$out"; then
    echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return
  fi
  echo "ok    $what"
}

gen "$tmp/all.json"                  || exit 2
gen "$tmp/extra.json"    extra       || exit 2
gen "$tmp/short.json"    drop-one    || exit 2
gen "$tmp/wrongver.json" wrong-ver   || exit 2
gen "$tmp/dropped.json"  drop-pinned || exit 2
printf 'this is not json\n' > "$tmp/broken.json"

expect pass "every advisory allowed and every allowance used passes"            ""                          "$tmp/all.json"
expect fail "an advisory the allow list does not name fails"                    "not in the allow list"     "$tmp/extra.json"
expect fail "an allowance that matches nothing fails"                           "no longer match anything"  "$tmp/short.json"
expect fail "a version-pinned allowance at an unnamed version fails"            "not in the allow list"     "$tmp/wrongver.json"
expect fail "a version-pinned allowance whose named pair is gone fails"         "no longer"                 "$tmp/dropped.json"
expect fail "a document that does not parse fails as unaudited, not as clean"   "Could not be audited"      "$tmp/broken.json"

if [ $fails -ne 0 ]; then echo "check-advisories-selftest: $fails failure(s)"; exit 1; fi
echo "check-advisories-selftest: the check passes a reviewed tree and fails each way it should."