#!/bin/bash
# Re-creates ext/ — the format-probe vectors written by real encoders — and provenance.json.
#
#   TOOLS=<dir> bash tools/make_external.sh          (from wai/format-probe-conformance)
#
# Every input is a synthetic picture tools/yuv.py writes; no captured content enters. Every
# encoder and decoder is open source under a permissive licence, at the revision pinned
# below, and is built in $TOOLS/src/<key> (the build lines below are the ones used). The
# script refuses to run against another revision. Each vector is decoded again by a
# decoder, and each JPEG XS vector by two independent decoders and by one of them along two
# code paths; provenance.json records every command, digest and comparison.
#
# Not run in CI (it needs the encoders, a C++ toolchain and the network); its outputs are
# committed, and format_probe_verify.py and wai-rs check them on every change.
set -euo pipefail
cd "$(dirname "$0")/.."
HERE=$(pwd)
: "${TOOLS:?set TOOLS to the directory holding the built encoders and decoders}"
S=$TOOLS/src
WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT

# key | repository | revision | licence (SPDX) | build
#   vvc_enc  https://github.com/fraunhoferhhi/vvenc.git  # allow-name: repository URL
#            9428ea8636ae7f443ecde89999d16b2dfc421524 (tag v1.14.0) BSD-3-Clause-Clear
#            cmake -S . -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build
#   vvc_dec  https://github.com/fraunhoferhhi/vvdec.git  # allow-name: repository URL
#            cb1a3efd88b2ebd1141f1a9ca87c774f05de9249 (tag v3.2.1) BSD-3-Clause-Clear
#            cmake -S . -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build
#   jxs      https://github.com/OpenVisualCloud/SVT-JPEG-XS.git  # allow-name: repository URL
#            f4720203534d6f5ea90f828a58814af5597461d7 BSD-2-Clause-Patent
#            cmake -S . -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build
#   jxs2     the crate oxideav-jpegxs 0.0.8 (crates.io), MIT, without default features  # allow-name: crate name
#   mvhevc   https://vcgit.hhi.fraunhofer.de/jvet/HTM.git  # allow-name: repository URL
#            7a38d7f5dce928ec456de048a07929acac2dfd6e (tag HTM-16.3) BSD-3-Clause
#            make -C build/linux HEVC_EXT=1 release (with -Werror removed from
#            build/linux/common/makefile.base: the 2018 sources warn under current compilers)
#   avs3_enc https://github.com/uavs3/uavs3e.git  # allow-name: repository URL
#            e1ff0f37a8d67814e1d650f6b49d495c49dde946 BSD-4-Clause
#            cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5
#   avs3_dec https://github.com/uavs3/uavs3d.git  # allow-name: repository URL
#            0e20d2c291853f196c68922a264bcd8471d75b68 BSD-3-Clause
#            cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCOMPILE_10BIT=0
pin() {
  local got
  got=$(git -C "$S/$1" rev-parse HEAD)
  [ "$got" = "$2" ] || { echo "$1 is at $got, not the pinned $2" >&2; exit 1; }
}
pin vvenc 9428ea8636ae7f443ecde89999d16b2dfc421524
pin vvdec cb1a3efd88b2ebd1141f1a9ca87c774f05de9249
pin SVT-JPEG-XS f4720203534d6f5ea90f828a58814af5597461d7
pin HTM 7a38d7f5dce928ec456de048a07929acac2dfd6e
pin uavs3e e1ff0f37a8d67814e1d650f6b49d495c49dde946
pin uavs3d 0e20d2c291853f196c68922a264bcd8471d75b68

VVENC=$S/vvenc/bin/release-static/vvencapp
VVDEC=$S/vvdec/bin/release-static/vvdecapp
JXSENC=$S/SVT-JPEG-XS/Bin/Release/SvtJpegxsEncApp
JXSDEC=$S/SVT-JPEG-XS/Bin/Release/SvtJpegxsDecApp
HTMENC=$S/HTM/bin/TAppEncoderStatic
HTMDEC=$S/HTM/bin/TAppDecoderStatic
AVS3ENC=$S/uavs3e/build/uavs3enc
AVS3DEC=$S/uavs3d/build/uavs3dec
JXS2=$TOOLS/jxs2/target/release/jxs2

# The second JPEG XS decoder: a program over the crate's component decode, writing each
# component plane as the crate returns it (one byte a sample at 8 bits, else two, LE).
if [ ! -x "$JXS2" ] || ! grep -q -- --unrestricted "$TOOLS/jxs2/src/main.rs"; then
  mkdir -p "$TOOLS/jxs2/src"
  cat > "$TOOLS/jxs2/Cargo.toml" <<'EOF'
[package]
name = "jxs2"
version = "0.0.0"
edition = "2021"
[dependencies]
oxideav-jpegxs = { version = "=0.0.8", default-features = false }
EOF
  cat > "$TOOLS/jxs2/src/main.rs" <<'EOF'
fn main() {
    let a: Vec<String> = std::env::args().collect();
    let mut bytes = std::fs::read(&a[1]).expect("input");
    if a.get(3).map(String::as_str) == Some("--unrestricted") {
        // Ppih and Plev rewritten to 0 (unrestricted): for a profile or level code this
        // decoder does not know. Neither field changes how the codestream decodes.
        let lcap = u16::from_be_bytes([bytes[4], bytes[5]]) as usize;
        let at = 4 + lcap + 8;
        bytes[at..at + 4].fill(0);
    }
    let c = oxideav_jpegxs::decode_components(&bytes).expect("decode");
    let mut out = Vec::new();
    for p in &c.planes {
        out.extend_from_slice(&p.data);
    }
    std::fs::write(&a[2], out).expect("output");
}
EOF
  (cd "$TOOLS/jxs2" && cargo build -q --release --target-dir "$TOOLS/jxs2/target")
fi

Y="python3 -I tools/yuv.py"
F="python3 -I tools/frame.py"
mkdir -p ext/vvc ext/hevc ext/mvhevc ext/jxs ext/jxs-video ext/avs3
LOG=$WORK/log.jsonl
: > "$LOG"
note() { # file, encoder key, input description, command
  python3 -I -c 'import json,sys; print(json.dumps({"file": sys.argv[1], "encoder": sys.argv[2], "input": sys.argv[3], "command": sys.argv[4]}))' "$@" >> "$LOG"
}

# ---- H.266 ----
$Y "$WORK/v64.yuv" 64 64 3 420 10
$VVENC -i "$WORK/v64.yuv" -s 64x64 -c yuv420_10 -r 30 -f 3 --preset faster -q 37 -t 1 -o ext/vvc/main10_64x64.266 >/dev/null
note vvc/main10_64x64.266 vvc_enc "64x64 4:2:0 10-bit, 3 frames" "vvencapp -s 64x64 -c yuv420_10 -r 30 -f 3 --preset faster -q 37 -t 1"
$Y "$WORK/v100.yuv" 100 58 2 420 10
$VVENC -i "$WORK/v100.yuv" -s 100x58 -c yuv420_10 -r 25 -f 2 --preset faster -q 40 -t 1 -o ext/vvc/main10_100x58.266 >/dev/null
note vvc/main10_100x58.266 vvc_enc "100x58 4:2:0 10-bit, 2 frames" "vvencapp -s 100x58 -c yuv420_10 -r 25 -f 2 --preset faster -q 40 -t 1"

# ---- MV-HEVC and its base layer ----
$Y "$WORK/l.yuv" 64 64 3 420 8 0
$Y "$WORK/r.yuv" 64 64 3 420 8 4
(cd "$WORK" && $HTMENC -c "$S/HTM/cfg/MV-HEVC/baseCfg_2view.cfg" --InputFile_0=l.yuv --InputFile_1=r.yuv \
  --SourceWidth=64 --SourceHeight=64 --FrameRate=30 --FramesToBeEncoded=3 --QP=37 \
  --BitstreamFile=mv.bit --ReconFile_0=rec0.yuv --ReconFile_1=rec1.yuv >/dev/null)
cp "$WORK/mv.bit" ext/mvhevc/2view_64x64.265
note mvhevc/2view_64x64.265 mvhevc "two views 64x64 4:2:0 8-bit, 3 frames each, the second shifted 4 columns" "TAppEncoder -c cfg/MV-HEVC/baseCfg_2view.cfg --SourceWidth=64 --SourceHeight=64 --FrameRate=30 --FramesToBeEncoded=3 --QP=37"
$F base ext/mvhevc/2view_64x64.265 ext/hevc/base_64x64.265
note hevc/base_64x64.265 mvhevc "the base layer (nuh_layer_id 0) of mvhevc/2view_64x64.265" "tools/frame.py base"

# ---- JPEG XS ----
$Y "$WORK/j422.yuv" 64 64 1 422 10
$JXSENC -i "$WORK/j422.yuv" -w 64 -h 64 --colour-format yuv422 --input-depth 10 --bpp 3 -n 1 -b ext/jxs/main422_10_64x64.jxs --no-progress 1 >/dev/null
note jxs/main422_10_64x64.jxs jxs "64x64 4:2:2 10-bit" "SvtJpegxsEncApp -w 64 -h 64 --colour-format yuv422 --input-depth 10 --bpp 3 -n 1"
$Y "$WORK/jrgb.yuv" 64 64 1 rgb 8
$JXSENC -i "$WORK/jrgb.yuv" -w 64 -h 64 --colour-format rgb --input-depth 8 --color-transform 1 --profile latency --bpp 4 -n 1 -b ext/jxs/rgb_rct_8_64x64.jxs --no-progress 1 >/dev/null
note jxs/rgb_rct_8_64x64.jxs jxs "64x64 R, G, B planes, 8-bit" "SvtJpegxsEncApp -w 64 -h 64 --colour-format rgb --input-depth 8 --color-transform 1 --profile latency --bpp 4 -n 1"
$Y "$WORK/j444.yuv" 48 32 1 444 8
$JXSENC -i "$WORK/j444.yuv" -w 48 -h 32 --colour-format yuv444 --input-depth 8 --lossless 1 --coding-raw 0 -n 1 -b ext/jxs/lossless_444_8_48x32.jxs --no-progress 1 >/dev/null
note jxs/lossless_444_8_48x32.jxs jxs "48x32 4:4:4 8-bit" "SvtJpegxsEncApp -w 48 -h 32 --colour-format yuv444 --input-depth 8 --lossless 1 --coding-raw 0 -n 1"
$Y "$WORK/j420.yuv" 64 48 3 420 8
$JXSENC -i "$WORK/j420.yuv" -w 64 -h 48 --colour-format yuv420 --input-depth 8 --bpp 3 -n 3 -b "$WORK/j420.cs" --no-progress 1 >/dev/null
$F jxsvideo ext/jxs-video/main420_8_64x48_3f.jxsv 64 48 30 1 "$WORK/j420.cs"
note jxs-video/main420_8_64x48_3f.jxsv jxs "64x48 4:2:0 8-bit, 3 frames at 30/1, in wai.video.jxs's framing" "SvtJpegxsEncApp -w 64 -h 48 --colour-format yuv420 --input-depth 8 --bpp 3 -n 3; tools/frame.py jxsvideo"
python3 -I -c 'import struct,sys; b=open(sys.argv[1],"rb").read(); n=struct.unpack("<I",b[:4])[0]; at=20
for i in range(n):
    l=struct.unpack("<I",b[at:at+4])[0]; open("%s/frame%d.jxs"%(sys.argv[2],i),"wb").write(b[at+4:at+4+l]); at+=4+l' ext/jxs-video/main420_8_64x48_3f.jxsv "$WORK"

# ---- AVS3 video ----
$Y "$WORK/a.yuv" 128 64 3 420 8
# (from $WORK: the encoder writes a psnr.txt beside it)
(cd "$WORK" && $AVS3ENC -i a.yuv -w 128 -h 64 -f 3 -q 40 --fps_num 30 --fps_den 1 -d 8 -v 0 -o "$HERE/ext/avs3/main_128x64.avs3" >/dev/null)
note avs3/main_128x64.avs3 avs3_enc "128x64 4:2:0 8-bit, 3 frames" "uavs3enc -w 128 -h 64 -f 3 -q 40 --fps_num 30 --fps_den 1 -d 8"

# ---- Decodes and comparisons ----
DEC=$WORK/dec.jsonl
: > "$DEC"
dec() { # file, decoder key, output file
  python3 -I -c 'import hashlib,json,sys; d=open(sys.argv[3],"rb").read(); print(json.dumps({"file": sys.argv[1], "decoder": sys.argv[2], "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()}))' "$@" >> "$DEC"
}
for f in ext/vvc/*.266; do $VVDEC -b "$f" -o "$WORK/d.yuv" >/dev/null; dec "${f#ext/}" vvc_dec "$WORK/d.yuv"; done
(cd "$WORK" && $HTMDEC -b "$HERE/ext/mvhevc/2view_64x64.265" -o dmv.yuv >/dev/null)
for o in "$WORK"/dmv*.yuv; do dec "mvhevc/2view_64x64.265" "mvhevc:$(basename "$o")" "$o"; done
(cd "$WORK" && $HTMDEC -b "$HERE/ext/hevc/base_64x64.265" -o dbase.yuv >/dev/null)
for o in "$WORK"/dbase*.yuv; do dec "hevc/base_64x64.265" "mvhevc:$(basename "$o")" "$o"; done
$AVS3DEC -i ext/avs3/main_128x64.avs3 -o "$WORK/da.yuv" -l 0 >/dev/null
dec avs3/main_128x64.avs3 avs3_dec "$WORK/da.yuv"
jx() { # codestream, name
  $JXSDEC -i "$1" -o "$WORK/s.yuv" --asm c -v 0 >/dev/null; dec "$2" jxs:c "$WORK/s.yuv"
  $JXSDEC -i "$1" -o "$WORK/s.yuv" -v 0 >/dev/null; dec "$2" jxs:simd "$WORK/s.yuv"
  # The second decoder refuses a profile or level code it does not know (MLS.12, added by
  # ISO/IEC 21122-2:2022/Amd 1; the level codes 0x04 and 0x2C); then it decodes a copy whose
  # Ppih and Plev are 0, and the record says so.
  if "$JXS2" "$1" "$WORK/s2.yuv" 2>/dev/null; then dec "$2" jxs2 "$WORK/s2.yuv"
  else "$JXS2" "$1" "$WORK/s2.yuv" --unrestricted; dec "$2" jxs2:unrestricted "$WORK/s2.yuv"; fi
}
for f in ext/jxs/*.jxs; do jx "$f" "${f#ext/}"; done
# The lossless codestream's source picture, which its decode must reproduce.
dec jxs/lossless_444_8_48x32.jxs source "$WORK/j444.yuv"
for i in 0 1 2; do jx "$WORK/frame$i.jxs" "jxs-video/main420_8_64x48_3f.jxsv#$i"; done

python3 -I tools/provenance.py "$LOG" "$DEC" > provenance.json