diff --git a/.github/workflows/repo-checks.yml b/.github/workflows/repo-checks.yml index 7df9bf358..84c6a50d7 100644 --- a/.github/workflows/repo-checks.yml +++ b/.github/workflows/repo-checks.yml @@ -168,6 +168,14 @@ jobs: fi bash scripts/check-public-names.sh + # ── Every vector of a corpus that keeps licence records has one ────────── + - name: "corpus licences: the check fails closed (self-test)" + if: ${{ !cancelled() }} + run: bash scripts/check-wai-corpus-licences-selftest.sh + - name: "corpus licences: every vector has a record and a licence that permits redistribution" + if: ${{ !cancelled() }} + run: python3 -I scripts/check-wai-corpus-licences.py + # ── No library source file is unreachable from its crate root ─────────── # An orphan `.rs` (one no `mod`, `#[path]` or `include!` names) is never compiled, # so nothing else in this repository can see it. Both directions, before trusting diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index 5d9fb631b..af224f22f 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -114,6 +114,8 @@ on: - 'wai/wai-web/**' # The audio provenance check below reads the corpora above with this tool. - 'wai/tools/wai_audio_provenance_check.py' + # The trained int_synth vector's picture check below runs this tool. + - 'wai/tools/wai_int_synth_real.py' - '.github/workflows/wai-conformance.yml' # The steps of the independent verifiers, the corpora of cargo-classes, and the script # that derives this workflow's lists from them (scripts/wai-conformance-lists.py). @@ -610,6 +612,43 @@ jobs: else echo "audio provenance: SKIPPED — no python3 on this host" fi + # neural-conformance's trained vector codes the picture its tool generates: the tool + # regenerates that picture from the seed in the vector's meta.json, with the Python + # standard library alone, and requires the vector's reference decode, whose SHA-256 + # is the corpus's digest for it, to reconstruct the picture at the PSNR meta.json + # records, and the vector's licence record to state the seed and streams meta.json + # records. One leg: the generator is integer arithmetic. + - name: The trained int_synth vector codes the picture its tool generates (macOS) + if: matrix.os == 'macos' + shell: bash + working-directory: wai + run: python3 -I tools/wai_int_synth_real.py --check neural-conformance/vectors/photo + # The same check on a copy of the corpus whose licence record gives the training + # pictures as streams 1 to 64 instead of meta.json's 1 to 512: it must fail. + - name: The trained int_synth vector's check fails on a licence record that misstates its training pictures (macOS) + if: matrix.os == 'macos' + shell: bash + working-directory: wai + run: | + set -uo pipefail + t=$(mktemp -d) + trap 'rm -rf "$t"' EXIT + mkdir -p "$t/vectors" + cp -R neural-conformance/vectors/photo "$t/vectors/" + cp neural-conformance/sha256.json neural-conformance/licences.json "$t/" + python3 -I - "$t/licences.json" <<'P' + import json, sys + d = json.load(open(sys.argv[1])) + tr = d["vectors"]["photo"]["training"] + assert tr["input"].endswith("streams 1 to 512"), tr["input"] + tr["input"] = tr["input"][: -len("512")] + "64" + json.dump(d, open(sys.argv[1], "w")) + P + s=0 + python3 -I tools/wai_int_synth_real.py --check "$t/vectors/photo" > "$t/out" 2>&1 || s=$? + cat "$t/out" + test "$s" = 1 + grep -qF "streams 1 to 64'; meta.json says" "$t/out" # OER's second implementation is portable C, not Rust. Rebuild it from source and # require its output to match the pinned corpus byte for byte: the kernel is # deterministic, and wai_oer_vectors above has already verified the pinned copy, diff --git a/.github/workflows/wai-ops.yml b/.github/workflows/wai-ops.yml index 024e605da..5c18c8084 100644 --- a/.github/workflows/wai-ops.yml +++ b/.github/workflows/wai-ops.yml @@ -174,14 +174,14 @@ jobs: bounds 0 neural-conformance/vectors/facade/int_synth.json --capability wai.neural.int_synth --profile acc32 \ --check $C/c6a362696d61605931ef4da98ae630362dcd73ea6be7864db6e61d6c5abbcd24.json bounds 0 neural-conformance/vectors/photo/int_synth.json --capability wai.neural.int_synth --profile acc64 \ - --check $C/50868e72f5f1599569c413ed1f67ab8610fa0b3c17919162cbf370f49275125c.json + --check $C/7c15df3437b7f9a47aeec9107c782149e308b72f81c7a3981999769ab9eda1e5.json # With --witness, a node-local graph for each failing node: photo's node 6, whose # input attains convT.acc's upper end. With --advise, the clamps that would let # the overflow set meet acc64. bounds 1 neural-conformance/vectors/photo/int_synth.json --capability wai.neural.int_synth --profile acc32 \ --witness "$T/witness" test -s "$T/witness/synth-6.json" - grep -q '"acc":"26172239136"' "$T/witness/synth-6.json" + grep -q '"acc":"3092313901"' "$T/witness/synth-6.json" bounds 1 ops-conformance/sets/overflow_acc64/int_synth.json --capability wai.neural.int_synth --advise acc64 # A certificate that is not the committed one is exit 3. bounds 3 neural-conformance/vectors/photo/int_synth.json --capability wai.neural.int_synth \ diff --git a/scripts/check-wai-corpus-licences-selftest.sh b/scripts/check-wai-corpus-licences-selftest.sh new file mode 100755 index 000000000..c122c9612 --- /dev/null +++ b/scripts/check-wai-corpus-licences-selftest.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# check-wai-corpus-licences-selftest.sh — check-wai-corpus-licences.py can fail. +# +# A check that cannot fail is decoration. This builds a scratch tree shaped like the +# repository (wai/neural-conformance with vector directories, its two hash manifests and +# licences.json, and a generator below wai/) whose every vector has a licence record, and +# asserts that the check passes it, and fails each way a record can be missing, name a +# licence that does not permit redistribution, or leave out what its content needs. +# Nothing here reads the real tree, and nothing needs a network. +set -uo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +check="$here/check-wai-corpus-licences.py" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fails=0 + +# mk : a corpus that holds. `seeded` and `trained` are generated, `third` is a +# third-party picture under CC-BY-4.0; `trained` is marked trained in expected_hashes.json. +mk() { + local c="$1/wai/neural-conformance" + mkdir -p "$c/vectors/seeded" "$c/vectors/trained" "$c/vectors/third" "$1/wai/tools" + printf '# a generator\n' > "$1/wai/tools/gen.py" + cat > "$c/expected_hashes.json" <<'J' +{"seeded": {"rgb_hash": "00"}, "trained": {"trained": true, "rgb_hash": "01"}, "third": {"rgb_hash": "02"}} +J + cat > "$c/sha256.json" <<'J' +{"seeded": {"sha256": "00"}, "trained": {"sha256": "01"}, "third": {"sha256": "02"}} +J + cat > "$c/licences.json" <<'J' +{"format": "wai-corpus-licences/1", "vectors": { + "seeded": {"licence": "Apache-2.0", "content": "generated", "generator": "tools/gen.py", "input": "seed 1"}, + "trained": {"licence": "Apache-2.0", "content": "generated", "generator": "tools/gen.py", "input": "seed 2, stream 0", + "training": {"content": "generated", "generator": "tools/gen.py", "input": "seed 2, streams 1 to 8"}}, + "third": {"licence": "CC-BY-4.0", "content": "third-party", "source": "a picture and its licence page", "generator": "tools/gen.py"} +}} +J +} + +# edit +edit() { + python3 -I - "$1/wai/neural-conformance/licences.json" "$2" <<'P' +import json, sys +p, stmt = sys.argv[1], sys.argv[2] +d = json.load(open(p)) +v = d["vectors"] +exec(stmt) +json.dump(d, open(p, "w")) +P +} + +want() { # want