7d61c2f8f1
External vectors written by permissively licensed encoders at pinned revisions from synthetic pictures (tools/make_external.sh, provenance.json), generated vectors and negative controls, the facts each encoder was asked for, the LCEVC streams fetch.sh fetches by digest, and format_probe_verify.py, written from the two texts. Co-Authored-By: Claude Opus 5.5 <[email protected]>
179 lines
11 KiB
Shell
179 lines
11 KiB
Shell
#!/bin/bash
|
|
# Re-creates ext/ — the format-probe vectors written by real encoders — and provenance.json.
|
|
#
|
|
# TOOLS=<dir> bash tools/make_external.sh (from wai/format-probe-conformance)
|
|
#
|
|
# Every input is a synthetic picture tools/yuv.py writes; no captured content enters. Every
|
|
# encoder and decoder is open source under a permissive licence, at the revision pinned
|
|
# below, and is built in $TOOLS/src/<key> (the build lines below are the ones used). The
|
|
# script refuses to run against another revision. Each vector is decoded again by a
|
|
# decoder, and each JPEG XS vector by two independent decoders and by one of them along two
|
|
# code paths; provenance.json records every command, digest and comparison.
|
|
#
|
|
# Not run in CI (it needs the encoders, a C++ toolchain and the network); its outputs are
|
|
# committed, and format_probe_verify.py and wai-rs check them on every change.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
HERE=$(pwd)
|
|
: "${TOOLS:?set TOOLS to the directory holding the built encoders and decoders}"
|
|
S=$TOOLS/src
|
|
WORK=$(mktemp -d)
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
# key | repository | revision | licence (SPDX) | build
|
|
# vvc_enc https://github.com/fraunhoferhhi/vvenc.git # allow-name: repository URL
|
|
# 9428ea8636ae7f443ecde89999d16b2dfc421524 (tag v1.14.0) BSD-3-Clause-Clear
|
|
# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build
|
|
# vvc_dec https://github.com/fraunhoferhhi/vvdec.git # allow-name: repository URL
|
|
# cb1a3efd88b2ebd1141f1a9ca87c774f05de9249 (tag v3.2.1) BSD-3-Clause-Clear
|
|
# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build
|
|
# jxs https://github.com/OpenVisualCloud/SVT-JPEG-XS.git # allow-name: repository URL
|
|
# f4720203534d6f5ea90f828a58814af5597461d7 BSD-2-Clause-Patent
|
|
# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build
|
|
# jxs2 the crate oxideav-jpegxs 0.0.8 (crates.io), MIT, without default features # allow-name: crate name
|
|
# mvhevc https://vcgit.hhi.fraunhofer.de/jvet/HTM.git # allow-name: repository URL
|
|
# 7a38d7f5dce928ec456de048a07929acac2dfd6e (tag HTM-16.3) BSD-3-Clause
|
|
# make -C build/linux HEVC_EXT=1 release (with -Werror removed from
|
|
# build/linux/common/makefile.base: the 2018 sources warn under current compilers)
|
|
# avs3_enc https://github.com/uavs3/uavs3e.git # allow-name: repository URL
|
|
# e1ff0f37a8d67814e1d650f6b49d495c49dde946 BSD-4-Clause
|
|
# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5
|
|
# avs3_dec https://github.com/uavs3/uavs3d.git # allow-name: repository URL
|
|
# 0e20d2c291853f196c68922a264bcd8471d75b68 BSD-3-Clause
|
|
# cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCOMPILE_10BIT=0
|
|
pin() {
|
|
local got
|
|
got=$(git -C "$S/$1" rev-parse HEAD)
|
|
[ "$got" = "$2" ] || { echo "$1 is at $got, not the pinned $2" >&2; exit 1; }
|
|
}
|
|
pin vvenc 9428ea8636ae7f443ecde89999d16b2dfc421524
|
|
pin vvdec cb1a3efd88b2ebd1141f1a9ca87c774f05de9249
|
|
pin SVT-JPEG-XS f4720203534d6f5ea90f828a58814af5597461d7
|
|
pin HTM 7a38d7f5dce928ec456de048a07929acac2dfd6e
|
|
pin uavs3e e1ff0f37a8d67814e1d650f6b49d495c49dde946
|
|
pin uavs3d 0e20d2c291853f196c68922a264bcd8471d75b68
|
|
|
|
VVENC=$S/vvenc/bin/release-static/vvencapp
|
|
VVDEC=$S/vvdec/bin/release-static/vvdecapp
|
|
JXSENC=$S/SVT-JPEG-XS/Bin/Release/SvtJpegxsEncApp
|
|
JXSDEC=$S/SVT-JPEG-XS/Bin/Release/SvtJpegxsDecApp
|
|
HTMENC=$S/HTM/bin/TAppEncoderStatic
|
|
HTMDEC=$S/HTM/bin/TAppDecoderStatic
|
|
AVS3ENC=$S/uavs3e/build/uavs3enc
|
|
AVS3DEC=$S/uavs3d/build/uavs3dec
|
|
JXS2=$TOOLS/jxs2/target/release/jxs2
|
|
|
|
# The second JPEG XS decoder: a program over the crate's component decode, writing each
|
|
# component plane as the crate returns it (one byte a sample at 8 bits, else two, LE).
|
|
if [ ! -x "$JXS2" ] || ! grep -q -- --unrestricted "$TOOLS/jxs2/src/main.rs"; then
|
|
mkdir -p "$TOOLS/jxs2/src"
|
|
cat > "$TOOLS/jxs2/Cargo.toml" <<'EOF'
|
|
[package]
|
|
name = "jxs2"
|
|
version = "0.0.0"
|
|
edition = "2021"
|
|
[dependencies]
|
|
oxideav-jpegxs = { version = "=0.0.8", default-features = false }
|
|
EOF
|
|
cat > "$TOOLS/jxs2/src/main.rs" <<'EOF'
|
|
fn main() {
|
|
let a: Vec<String> = std::env::args().collect();
|
|
let mut bytes = std::fs::read(&a[1]).expect("input");
|
|
if a.get(3).map(String::as_str) == Some("--unrestricted") {
|
|
// Ppih and Plev rewritten to 0 (unrestricted): for a profile or level code this
|
|
// decoder does not know. Neither field changes how the codestream decodes.
|
|
let lcap = u16::from_be_bytes([bytes[4], bytes[5]]) as usize;
|
|
let at = 4 + lcap + 8;
|
|
bytes[at..at + 4].fill(0);
|
|
}
|
|
let c = oxideav_jpegxs::decode_components(&bytes).expect("decode");
|
|
let mut out = Vec::new();
|
|
for p in &c.planes {
|
|
out.extend_from_slice(&p.data);
|
|
}
|
|
std::fs::write(&a[2], out).expect("output");
|
|
}
|
|
EOF
|
|
(cd "$TOOLS/jxs2" && cargo build -q --release --target-dir "$TOOLS/jxs2/target")
|
|
fi
|
|
|
|
Y="python3 -I tools/yuv.py"
|
|
F="python3 -I tools/frame.py"
|
|
mkdir -p ext/vvc ext/hevc ext/mvhevc ext/jxs ext/jxs-video ext/avs3
|
|
LOG=$WORK/log.jsonl
|
|
: > "$LOG"
|
|
note() { # file, encoder key, input description, command
|
|
python3 -I -c 'import json,sys; print(json.dumps({"file": sys.argv[1], "encoder": sys.argv[2], "input": sys.argv[3], "command": sys.argv[4]}))' "$@" >> "$LOG"
|
|
}
|
|
|
|
# ---- H.266 ----
|
|
$Y "$WORK/v64.yuv" 64 64 3 420 10
|
|
$VVENC -i "$WORK/v64.yuv" -s 64x64 -c yuv420_10 -r 30 -f 3 --preset faster -q 37 -t 1 -o ext/vvc/main10_64x64.266 >/dev/null
|
|
note vvc/main10_64x64.266 vvc_enc "64x64 4:2:0 10-bit, 3 frames" "vvencapp -s 64x64 -c yuv420_10 -r 30 -f 3 --preset faster -q 37 -t 1"
|
|
$Y "$WORK/v100.yuv" 100 58 2 420 10
|
|
$VVENC -i "$WORK/v100.yuv" -s 100x58 -c yuv420_10 -r 25 -f 2 --preset faster -q 40 -t 1 -o ext/vvc/main10_100x58.266 >/dev/null
|
|
note vvc/main10_100x58.266 vvc_enc "100x58 4:2:0 10-bit, 2 frames" "vvencapp -s 100x58 -c yuv420_10 -r 25 -f 2 --preset faster -q 40 -t 1"
|
|
|
|
# ---- MV-HEVC and its base layer ----
|
|
$Y "$WORK/l.yuv" 64 64 3 420 8 0
|
|
$Y "$WORK/r.yuv" 64 64 3 420 8 4
|
|
(cd "$WORK" && $HTMENC -c "$S/HTM/cfg/MV-HEVC/baseCfg_2view.cfg" --InputFile_0=l.yuv --InputFile_1=r.yuv \
|
|
--SourceWidth=64 --SourceHeight=64 --FrameRate=30 --FramesToBeEncoded=3 --QP=37 \
|
|
--BitstreamFile=mv.bit --ReconFile_0=rec0.yuv --ReconFile_1=rec1.yuv >/dev/null)
|
|
cp "$WORK/mv.bit" ext/mvhevc/2view_64x64.265
|
|
note mvhevc/2view_64x64.265 mvhevc "two views 64x64 4:2:0 8-bit, 3 frames each, the second shifted 4 columns" "TAppEncoder -c cfg/MV-HEVC/baseCfg_2view.cfg --SourceWidth=64 --SourceHeight=64 --FrameRate=30 --FramesToBeEncoded=3 --QP=37"
|
|
$F base ext/mvhevc/2view_64x64.265 ext/hevc/base_64x64.265
|
|
note hevc/base_64x64.265 mvhevc "the base layer (nuh_layer_id 0) of mvhevc/2view_64x64.265" "tools/frame.py base"
|
|
|
|
# ---- JPEG XS ----
|
|
$Y "$WORK/j422.yuv" 64 64 1 422 10
|
|
$JXSENC -i "$WORK/j422.yuv" -w 64 -h 64 --colour-format yuv422 --input-depth 10 --bpp 3 -n 1 -b ext/jxs/main422_10_64x64.jxs --no-progress 1 >/dev/null
|
|
note jxs/main422_10_64x64.jxs jxs "64x64 4:2:2 10-bit" "SvtJpegxsEncApp -w 64 -h 64 --colour-format yuv422 --input-depth 10 --bpp 3 -n 1"
|
|
$Y "$WORK/jrgb.yuv" 64 64 1 rgb 8
|
|
$JXSENC -i "$WORK/jrgb.yuv" -w 64 -h 64 --colour-format rgb --input-depth 8 --color-transform 1 --profile latency --bpp 4 -n 1 -b ext/jxs/rgb_rct_8_64x64.jxs --no-progress 1 >/dev/null
|
|
note jxs/rgb_rct_8_64x64.jxs jxs "64x64 R, G, B planes, 8-bit" "SvtJpegxsEncApp -w 64 -h 64 --colour-format rgb --input-depth 8 --color-transform 1 --profile latency --bpp 4 -n 1"
|
|
$Y "$WORK/j444.yuv" 48 32 1 444 8
|
|
$JXSENC -i "$WORK/j444.yuv" -w 48 -h 32 --colour-format yuv444 --input-depth 8 --lossless 1 --coding-raw 0 -n 1 -b ext/jxs/lossless_444_8_48x32.jxs --no-progress 1 >/dev/null
|
|
note jxs/lossless_444_8_48x32.jxs jxs "48x32 4:4:4 8-bit" "SvtJpegxsEncApp -w 48 -h 32 --colour-format yuv444 --input-depth 8 --lossless 1 --coding-raw 0 -n 1"
|
|
$Y "$WORK/j420.yuv" 64 48 3 420 8
|
|
$JXSENC -i "$WORK/j420.yuv" -w 64 -h 48 --colour-format yuv420 --input-depth 8 --bpp 3 -n 3 -b "$WORK/j420.cs" --no-progress 1 >/dev/null
|
|
$F jxsvideo ext/jxs-video/main420_8_64x48_3f.jxsv 64 48 30 1 "$WORK/j420.cs"
|
|
note jxs-video/main420_8_64x48_3f.jxsv jxs "64x48 4:2:0 8-bit, 3 frames at 30/1, in wai.video.jxs's framing" "SvtJpegxsEncApp -w 64 -h 48 --colour-format yuv420 --input-depth 8 --bpp 3 -n 3; tools/frame.py jxsvideo"
|
|
python3 -I -c 'import struct,sys; b=open(sys.argv[1],"rb").read(); n=struct.unpack("<I",b[:4])[0]; at=20
|
|
for i in range(n):
|
|
l=struct.unpack("<I",b[at:at+4])[0]; open("%s/frame%d.jxs"%(sys.argv[2],i),"wb").write(b[at+4:at+4+l]); at+=4+l' ext/jxs-video/main420_8_64x48_3f.jxsv "$WORK"
|
|
|
|
# ---- AVS3 video ----
|
|
$Y "$WORK/a.yuv" 128 64 3 420 8
|
|
# (from $WORK: the encoder writes a psnr.txt beside it)
|
|
(cd "$WORK" && $AVS3ENC -i a.yuv -w 128 -h 64 -f 3 -q 40 --fps_num 30 --fps_den 1 -d 8 -v 0 -o "$HERE/ext/avs3/main_128x64.avs3" >/dev/null)
|
|
note avs3/main_128x64.avs3 avs3_enc "128x64 4:2:0 8-bit, 3 frames" "uavs3enc -w 128 -h 64 -f 3 -q 40 --fps_num 30 --fps_den 1 -d 8"
|
|
|
|
# ---- Decodes and comparisons ----
|
|
DEC=$WORK/dec.jsonl
|
|
: > "$DEC"
|
|
dec() { # file, decoder key, output file
|
|
python3 -I -c 'import hashlib,json,sys; d=open(sys.argv[3],"rb").read(); print(json.dumps({"file": sys.argv[1], "decoder": sys.argv[2], "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()}))' "$@" >> "$DEC"
|
|
}
|
|
for f in ext/vvc/*.266; do $VVDEC -b "$f" -o "$WORK/d.yuv" >/dev/null; dec "${f#ext/}" vvc_dec "$WORK/d.yuv"; done
|
|
(cd "$WORK" && $HTMDEC -b "$HERE/ext/mvhevc/2view_64x64.265" -o dmv.yuv >/dev/null)
|
|
for o in "$WORK"/dmv*.yuv; do dec "mvhevc/2view_64x64.265" "mvhevc:$(basename "$o")" "$o"; done
|
|
(cd "$WORK" && $HTMDEC -b "$HERE/ext/hevc/base_64x64.265" -o dbase.yuv >/dev/null)
|
|
for o in "$WORK"/dbase*.yuv; do dec "hevc/base_64x64.265" "mvhevc:$(basename "$o")" "$o"; done
|
|
$AVS3DEC -i ext/avs3/main_128x64.avs3 -o "$WORK/da.yuv" -l 0 >/dev/null
|
|
dec avs3/main_128x64.avs3 avs3_dec "$WORK/da.yuv"
|
|
jx() { # codestream, name
|
|
$JXSDEC -i "$1" -o "$WORK/s.yuv" --asm c -v 0 >/dev/null; dec "$2" jxs:c "$WORK/s.yuv"
|
|
$JXSDEC -i "$1" -o "$WORK/s.yuv" -v 0 >/dev/null; dec "$2" jxs:simd "$WORK/s.yuv"
|
|
# The second decoder refuses a profile or level code it does not know (MLS.12, added by
|
|
# ISO/IEC 21122-2:2022/Amd 1; the level codes 0x04 and 0x2C); then it decodes a copy whose
|
|
# Ppih and Plev are 0, and the record says so.
|
|
if "$JXS2" "$1" "$WORK/s2.yuv" 2>/dev/null; then dec "$2" jxs2 "$WORK/s2.yuv"
|
|
else "$JXS2" "$1" "$WORK/s2.yuv" --unrestricted; dec "$2" jxs2:unrestricted "$WORK/s2.yuv"; fi
|
|
}
|
|
for f in ext/jxs/*.jxs; do jx "$f" "${f#ext/}"; done
|
|
# The lossless codestream's source picture, which its decode must reproduce.
|
|
dec jxs/lossless_444_8_48x32.jxs source "$WORK/j444.yuv"
|
|
for i in 0 1 2; do jx "$WORK/frame$i.jxs" "jxs-video/main420_8_64x48_3f.jxsv#$i"; done
|
|
|
|
python3 -I tools/provenance.py "$LOG" "$DEC" > provenance.json
|