open-standards/scripts/check-wai-corpus-licences-selftest.sh

#!/usr/bin/env bash
#
# check-wai-corpus-licences-selftest.sh — check-wai-corpus-licences.py can fail.
#
# A check that cannot fail is decoration. This builds a scratch tree shaped like the
# repository (wai/neural-conformance with vector directories, its two hash manifests and
# licences.json, wai/tensor-conformance with one vector and its licences.json, and a
# generator below wai/) whose every vector has a licence record, and
# asserts that the check passes it, and fails each way a record can be missing, name a
# licence that does not permit redistribution, or leave out what its content needs.
# Nothing here reads the real tree, and nothing needs a network.
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
check="$here/check-wai-corpus-licences.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
fails=0

# mk <root>: a corpus that holds. `seeded` and `trained` are generated, `third` is a
# third-party picture under CC-BY-4.0; `trained` is marked trained in expected_hashes.json.
mk() {
  local c="$1/wai/neural-conformance"
  mkdir -p "$c/vectors/seeded" "$c/vectors/trained" "$c/vectors/third" "$1/wai/tools"
  printf '# a generator\n' > "$1/wai/tools/gen.py"
  cat > "$c/expected_hashes.json" <<'J'
{"seeded": {"rgb_hash": "00"}, "trained": {"trained": true, "rgb_hash": "01"}, "third": {"rgb_hash": "02"}}
J
  cat > "$c/sha256.json" <<'J'
{"seeded": {"sha256": "00"}, "trained": {"sha256": "01"}, "third": {"sha256": "02"}}
J
  cat > "$c/licences.json" <<'J'
{"format": "wai-corpus-licences/1", "vectors": {
 "seeded": {"licence": "Apache-2.0", "content": "generated", "generator": "tools/gen.py", "input": "seed 1"},
 "trained": {"licence": "Apache-2.0", "content": "generated", "generator": "tools/gen.py", "input": "seed 2, stream 0",
             "training": {"content": "generated", "generator": "tools/gen.py", "input": "seed 2, streams 1 to 8"}},
 "third": {"licence": "CC-BY-4.0", "content": "third-party", "source": "a picture and its licence page", "generator": "tools/gen.py"}
}}
J
  # The check reads every corpus in its CORPORA, so the tree holds each of them.
  local t="$1/wai/tensor-conformance"
  mkdir -p "$t/vectors/frame"
  cat > "$t/licences.json" <<'J'
{"format": "wai-corpus-licences/1", "vectors": {
 "frame": {"licence": "Apache-2.0", "content": "generated", "generator": "tools/gen.py", "input": "seed 3"}
}}
J
}

# edit <root> <python statement on `d`, the parsed licences.json>
edit() {
  python3 -I - "$1/wai/neural-conformance/licences.json" "$2" <<'P'
import json, sys
p, stmt = sys.argv[1], sys.argv[2]
d = json.load(open(p))
v = d["vectors"]
exec(stmt)
json.dump(d, open(p, "w"))
P
}

want() { # want <expect-status> <root> <label> [<text the output must contain>]
  python3 -I "$check" --root "$2" > "$tmp/out" 2>&1
  got=$?
  if [ "$got" != "$1" ]; then
    echo "FAIL: $3 — expected exit $1, got $got"
    sed 's/^/    /' "$tmp/out"
    fails=$((fails + 1))
  elif [ -n "${4:-}" ] && ! grep -qF -- "$4" "$tmp/out"; then
    echo "FAIL: $3 — the output does not say: $4"
    sed 's/^/    /' "$tmp/out"
    fails=$((fails + 1))
  else
    echo "ok: $3"
  fi
}

case_() { # case_ <label> <python statement> <text the finding must contain>
  local r="$tmp/$((n += 1))"
  mk "$r"
  edit "$r" "$2"
  want 1 "$r" "$1" "$3"
}
n=0

r="$tmp/ok"; mk "$r"
want 0 "$r" "passes when every vector has a record" "wai/neural-conformance: CC-BY-4.0: third"

r="$tmp/nofile"; mk "$r"; rm "$r/wai/neural-conformance/licences.json"
want 1 "$r" "fails when the corpus has no licences.json" "no licences.json"

r="$tmp/nofile2"; mk "$r"; rm "$r/wai/tensor-conformance/licences.json"
want 1 "$r" "fails when the second corpus has no licences.json" "wai/tensor-conformance: no licences.json"

r="$tmp/newdir2"; mk "$r"; mkdir -p "$r/wai/tensor-conformance/vectors/added"
want 1 "$r" "fails on a vector directory of the second corpus with no record" "wai/tensor-conformance: vector added has no licence record"

r="$tmp/badjson"; mk "$r"; printf '{"format": ' > "$r/wai/neural-conformance/licences.json"
want 1 "$r" "fails when licences.json does not parse" "does not parse"

r="$tmp/newdir"; mk "$r"; mkdir -p "$r/wai/neural-conformance/vectors/added"
want 1 "$r" "fails on a vector directory with no record" "vector added has no licence record"

r="$tmp/newhash"; mk "$r"
python3 -I -c 'import json,sys; p=sys.argv[1]; d=json.load(open(p)); d["listed"]={"rgb_hash":"03"}; json.dump(d,open(p,"w"))' \
  "$r/wai/neural-conformance/expected_hashes.json"
want 1 "$r" "fails on a manifest entry with no record" "vector listed has no licence record"

case_ "fails on the wrong format" 'd["format"] = "wai-corpus-licences/0"' "not a wai-corpus-licences/1 document"
case_ "fails on a record with no vector" 'v["gone"] = dict(v["seeded"])' "gone: no directory vectors/gone"
case_ "fails on a record with no licence" 'del v["seeded"]["licence"]' "seeded: licence None is not one of"
case_ "fails on a licence that grants nothing" 'v["seeded"]["licence"] = "NOASSERTION"' "licence 'NOASSERTION' is not one of"
case_ "fails on a licence that is not SPDX's spelling" 'v["seeded"]["licence"] = "apache 2.0"' "licence 'apache 2.0' is not one of"
case_ "fails on unknown content" 'v["seeded"]["content"] = "synthetic"' "content 'synthetic' is neither"
case_ "fails on generated content with no generator" 'del v["seeded"]["generator"]' "generated content names no generator"
case_ "fails on a generator that does not exist" 'v["seeded"]["generator"] = "tools/missing.py"' "generator 'tools/missing.py' is not a file below wai/"
case_ "fails on generated content with no input" 'v["seeded"]["input"] = " "' "generated content names no input"
case_ "fails on generated content with a source" 'v["seeded"]["source"] = "a photograph"' "generated content has a source"
case_ "fails on third-party content with no source" 'del v["third"]["source"]' "third-party content names no source"
case_ "fails on a trained model with no training record" 'del v["trained"]["training"]' "a trained model with no training record"
case_ "fails on a training record on a model not marked trained" 'v["seeded"]["training"] = dict(v["trained"]["training"])' "a training record on a vector whose model is not marked trained"
case_ "fails on training data with no generator" 'del v["trained"]["training"]["generator"]' "trained training: generated content names no generator"
case_ "fails on third-party training data with no licence" \
  'v["trained"]["training"] = {"content": "third-party", "source": "photographs"}' "trained training: licence None is not one of"
case_ "fails on third-party training data under a licence that grants nothing" \
  'v["trained"]["training"] = {"content": "third-party", "source": "photographs", "licence": "unknown"}' "licence 'unknown' is not one of"
case_ "fails on an unknown member" 'v["seeded"]["licnece"] = "Apache-2.0"' "unknown member 'licnece'"
case_ "fails on an unknown member of a training record" 'v["trained"]["training"]["seed"] = 2' "trained training: unknown member 'seed'"

r="$tmp/tp-train"; mk "$r"
edit "$r" 'v["trained"]["training"] = {"content": "third-party", "source": "photographs and their licence pages", "licence": "CC0-1.0"}'
want 0 "$r" "passes third-party training data under a licence that permits redistribution"

if [ "$fails" != 0 ]; then
  echo "check-wai-corpus-licences-selftest: $fails case(s) failed."
  exit 1
fi
echo "check-wai-corpus-licences-selftest: all cases pass."