106 lines
6.5 KiB
Rust
106 lines
6.5 KiB
Rust
//! The measurement of extensions/opus-dred.md §6, on the draft's own test vectors
|
|
//! ([DRED-Test-Vectors], draft-ietf-mlcodec-opus-dred-07 §7.1): every record of every DRED bit
|
|
//! file decoded through the integer profile and through the floating-point network, each
|
|
//! compared with the reference features the draft ships, and with each other, by the draft's
|
|
//! feature comparison (§4.1) and by the figures §6 adds. And every packet of every packet file
|
|
//! that carries DRED, its DRED found and decoded both ways.
|
|
//!
|
|
//! `WAI_DRED_WEIGHTS` names the weights file and `WAI_DRED_VECTORS` the unpacked test vectors
|
|
//! (`live-conformance/dred/fetch.sh <dir> vectors`). Without them the test fails rather than
|
|
//! pass without measuring. It prints every figure, and holds the integer profile to the draft's
|
|
//! thresholds against the reference features and §6's figures to the extension's table.
|
|
|
|
use std::path::PathBuf;
|
|
|
|
use wai::dred_exact::float_ref::FloatModel;
|
|
use wai::dred_exact::measure::{self, Stats};
|
|
use wai::dred_exact::{packet, payload, weights, Model};
|
|
|
|
fn env(k: &str) -> String {
|
|
std::env::var(k).unwrap_or_else(|_| panic!("{k} must be set: bash wai/live-conformance/dred/fetch.sh <dir> vectors"))
|
|
}
|
|
|
|
fn fmt(s: &Stats) -> String {
|
|
format!(
|
|
"frames {} total {:.4} max {:.4} pitch {:.5} | max|d| {:.4} pitch max|d| {:.4} | band LSD mean {:.4} dB max {:.4} dB",
|
|
s.frames, s.draft.total, s.draft.max, s.draft.pitch, s.max_abs, s.max_abs_pitch, s.lsd_mean_db, s.lsd_max_db
|
|
)
|
|
}
|
|
|
|
#[test]
|
|
fn the_profile_against_the_floating_point_network() {
|
|
let wbytes = std::fs::read(env("WAI_DRED_WEIGHTS")).unwrap();
|
|
let layers = weights::pinned(&wbytes).unwrap();
|
|
let model = Model::from_layers(&layers).unwrap();
|
|
let float = FloatModel::new(&layers);
|
|
let int8 = FloatModel::int8_reference(&layers, &weights::parse(&wbytes).unwrap());
|
|
let dir = PathBuf::from(env("WAI_DRED_VECTORS")).join("dred_testvectors");
|
|
|
|
let (mut all_int, mut all_flt, mut all_ref, mut all_q8) = (Vec::new(), Vec::new(), Vec::new(), Vec::new());
|
|
for v in 1..=8 {
|
|
let bits = std::fs::read(dir.join(format!("vector{v}_dred.bit"))).unwrap();
|
|
let reference = measure::read_features(&std::fs::read(dir.join(format!("vector{v}_dred_dec.f32"))).unwrap()).unwrap();
|
|
let (mut int_rows, mut flt_rows, mut q8_rows) = (Vec::new(), Vec::new(), Vec::new());
|
|
for r in measure::read_dred_bits(&bits).unwrap() {
|
|
let p = measure::record_payload(&r).unwrap();
|
|
let n = r.chunks / 2;
|
|
int_rows.extend(measure::oldest_first(&measure::rows_f64(&model.decode(&p, n).unwrap())));
|
|
flt_rows.extend(measure::oldest_first(&float.decode(&p, n)));
|
|
q8_rows.extend(measure::oldest_first(&int8.decode(&p, n)));
|
|
}
|
|
assert_eq!(int_rows.len(), reference.len(), "vector {v}");
|
|
let ir = measure::stats(&reference, &int_rows);
|
|
println!("dred-measure vector{v} integer vs reference: {}", fmt(&ir));
|
|
println!("dred-measure vector{v} float vs reference: {}", fmt(&measure::stats(&reference, &flt_rows)));
|
|
println!("dred-measure vector{v} integer vs float: {}", fmt(&measure::stats(&flt_rows, &int_rows)));
|
|
println!("dred-measure vector{v} 8-bit vs reference: {}", fmt(&measure::stats(&reference, &q8_rows)));
|
|
assert!(ir.draft.passes(), "vector {v}: the integer profile passes the draft's feature test against its reference features");
|
|
// Informative: the reference implementation's own outputs on this machine, when
|
|
// `WAI_DRED_REFOUT` names a directory of them (`v<N>_float.f32` from a build with its
|
|
// float weights, `v<N>_default.f32` from its default build), compared with both decodes.
|
|
if let Ok(d) = std::env::var("WAI_DRED_REFOUT") {
|
|
for build in ["float", "default"] {
|
|
if let Some(r) = std::fs::read(PathBuf::from(&d).join(format!("v{v}_{build}.f32"))).ok().and_then(|b| measure::read_features(&b)) {
|
|
println!("dred-measure vector{v} reference {build} build vs test vector: {}", fmt(&measure::stats(&reference, &r)));
|
|
println!("dred-measure vector{v} integer vs reference {build} build: {}", fmt(&measure::stats(&r, &int_rows)));
|
|
println!("dred-measure vector{v} float vs reference {build} build: {}", fmt(&measure::stats(&r, &flt_rows)));
|
|
}
|
|
}
|
|
}
|
|
all_q8.extend(q8_rows);
|
|
all_int.extend(int_rows);
|
|
all_flt.extend(flt_rows);
|
|
all_ref.extend(reference);
|
|
}
|
|
let (ir, fr, iff) = (measure::stats(&all_ref, &all_int), measure::stats(&all_ref, &all_flt), measure::stats(&all_flt, &all_int));
|
|
println!("dred-measure all integer vs reference: {}", fmt(&ir));
|
|
println!("dred-measure all float vs reference: {}", fmt(&fr));
|
|
println!("dred-measure all integer vs float: {}", fmt(&iff));
|
|
let q8 = measure::stats(&all_ref, &all_q8);
|
|
println!("dred-measure all 8-bit vs reference: {}", fmt(&q8));
|
|
|
|
// The packet files: every packet that carries DRED, its DRED decoded both ways.
|
|
let (mut pk_int, mut pk_flt, mut with_dred, mut packets) = (Vec::new(), Vec::new(), 0usize, 0usize);
|
|
for v in 1..=8 {
|
|
let b = std::fs::read(dir.join(format!("vector{v}_opus.bit"))).unwrap();
|
|
for p in measure::read_packets(&b).unwrap().into_iter().flatten() {
|
|
packets += 1;
|
|
let Some((bytes, _)) = packet::find_dred(&p).unwrap() else { continue };
|
|
let pl = payload::parse(bytes, None).unwrap();
|
|
with_dred += 1;
|
|
pk_int.extend(measure::rows_f64(&model.decode(&pl, usize::MAX).unwrap()));
|
|
pk_flt.extend(float.decode(&pl, usize::MAX));
|
|
}
|
|
}
|
|
let pk = measure::stats(&pk_flt, &pk_int);
|
|
println!("dred-measure packets: {packets} received, {with_dred} carry DRED; integer vs float: {}", fmt(&pk));
|
|
assert!(with_dred > 0);
|
|
|
|
// The extension's §6 table states these figures, to the digits it prints.
|
|
let text = std::fs::read_to_string(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../extensions/opus-dred.md")).unwrap();
|
|
for (label, s) in [("integer, reference", &ir), ("float, reference", &fr), ("8-bit emulation, reference", &q8), ("integer, float", &iff), ("integer, float (packets)", &pk)] {
|
|
let row = format!("| {label} | {} | {:.4} | {:.4} | {:.5} | {:.4} | {:.4} | {:.4} |", s.frames, s.draft.total, s.draft.max, s.draft.pitch, s.max_abs, s.lsd_mean_db, s.lsd_max_db);
|
|
println!("{row}");
|
|
assert!(text.contains(&row), "extensions/opus-dred.md §6 states: {row}");
|
|
}
|
|
}
|