7d61c2f8f1
External vectors written by permissively licensed encoders at pinned revisions from synthetic pictures (tools/make_external.sh, provenance.json), generated vectors and negative controls, the facts each encoder was asked for, the LCEVC streams fetch.sh fetches by digest, and format_probe_verify.py, written from the two texts. Co-Authored-By: Claude Opus 5.5 <[email protected]>
50 lines
1.8 KiB
Python
50 lines
1.8 KiB
Python
"""Writes provenance.json from make_external.sh's logs: every external vector with its
|
|
digest, encoder, input and command, and every decode of it with its output's size and
|
|
digest. Where several decoders or code paths decode one JPEG XS codestream, it records
|
|
whether their outputs are byte-identical.
|
|
|
|
python3 -I provenance.py <encodes.jsonl> <decodes.jsonl>
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
|
|
def main(enc_log, dec_log):
|
|
here = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "ext")
|
|
vectors = []
|
|
for line in open(enc_log):
|
|
e = json.loads(line)
|
|
b = open(os.path.join(here, e["file"]), "rb").read()
|
|
e["bytes"] = len(b)
|
|
e["sha256"] = hashlib.sha256(b).hexdigest()
|
|
e["decodes"] = []
|
|
vectors.append(e)
|
|
by_file = {}
|
|
for line in open(dec_log):
|
|
d = json.loads(line)
|
|
by_file.setdefault(d["file"], []).append(d)
|
|
for e in vectors:
|
|
e["decodes"] = [{k: d[k] for k in ("decoder", "bytes", "sha256")} for d in by_file.get(e["file"], [])]
|
|
agreement = []
|
|
for f, ds in sorted(by_file.items()):
|
|
js = [d for d in ds if d["decoder"].startswith("jxs") or d["decoder"] == "source"]
|
|
if len(js) > 1:
|
|
agreement.append({
|
|
"codestream": f,
|
|
"outputs": {d["decoder"]: d["sha256"] for d in js},
|
|
"identical": len({d["sha256"] for d in js}) == 1,
|
|
})
|
|
out = {
|
|
"note": "Written by tools/make_external.sh; each encoder and decoder key is defined, with its repository, revision and licence, in that script's header.",
|
|
"vectors": vectors,
|
|
"jpeg_xs_decodes": agreement,
|
|
}
|
|
json.dump(out, sys.stdout, indent=1, sort_keys=True)
|
|
sys.stdout.write("\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(*sys.argv[1:])
|