//! `wai.delivery.audit` (extensions/stream-audit.md): the committed corpus //! `live-conformance/audit/vectors.json` regenerates byte for byte from its fixed seeds and is the //! only file there besides its README, every session and report in it gets the outcome it states, //! every reason the extension adds and every code it borrows refuses a vector, the accepted //! vectors are the cases §15 names, and the extension's §14 lists exactly the reasons the //! reference reports. #![cfg(feature = "stream_audit")] use std::collections::BTreeSet; use std::path::PathBuf; use wai::stream_audit::{corpus, Reason}; fn wai_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..") } fn read(p: &str) -> String { std::fs::read_to_string(wai_dir().join(p)).unwrap_or_else(|e| panic!("{p}: {e}")) } fn committed() -> String { read("live-conformance/audit/vectors.json") } fn doc() -> serde_json::Value { serde_json::from_str(&committed()).unwrap() } #[test] fn the_committed_corpus_regenerates_from_its_seeds() { assert!(committed() == corpus::generate(), "live-conformance/audit/vectors.json differs from `wai_stream_audit_vectors gen`"); let mut files: Vec = std::fs::read_dir(wai_dir().join("live-conformance/audit")).unwrap().flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(); files.sort(); assert_eq!(files, ["README.md", "vectors.json"]); } #[test] fn every_session_and_report_gets_its_stated_outcome() { let fails = corpus::verify(&committed()); assert!(fails.is_empty(), "{}", fails.join("\n")); } /// Every reason this extension adds, and every code it borrows, refuses a vector or is a claim's /// verdict; the accepted sessions and reports are §15's, with the figures each was written for. #[test] fn the_corpus_covers_every_reason_and_case() { let doc = doc(); let mut seen: BTreeSet = BTreeSet::new(); for section in ["sessions", "reports"] { for v in doc[section].as_array().unwrap() { if let Some(r) = v["outcome"]["reason"].as_str() { seen.insert(r.to_string()); } for c in v["outcome"]["claims"].as_array().into_iter().flatten() { seen.insert(c["verdict"].as_str().unwrap().to_string()); } } } for r in Reason::ALL { assert!(seen.contains(r.code()), "nothing in the corpus is refused {r}"); } for r in Reason::REUSED.iter().chain(Reason::VERDICTS.iter()) { assert!(seen.contains(*r), "nothing in the corpus is refused {r}"); } let session = |n: &str| doc["sessions"].as_array().unwrap().iter().find(|v| v["name"] == n).unwrap_or_else(|| panic!("no session {n}"))["outcome"].clone(); let live = session("live_two_origins"); assert_eq!(live["outcome"], "accept"); assert_eq!(live["presentation"], "live"); assert_eq!(live["claims"][0]["verdict"], "presented"); let viewable: Vec = live["claims"].as_array().unwrap().iter().map(|c| c["viewable"].as_bool().unwrap()).collect(); assert!(viewable.contains(&true) && viewable.contains(&false), "one ad in view, one half out of it"); assert!(live["repeats"]["objects"].as_u64().unwrap() >= 1, "a repeat"); assert_eq!(live["notes"], 2); let origins: BTreeSet<&str> = live["delivery"].as_array().unwrap().iter().map(|d| d["origin"].as_str().unwrap()).collect(); assert_eq!(origins.len(), 2, "two origins"); assert_eq!(live["energy"][0]["class"], "Estimator"); assert_eq!(live["energy"][1]["class"], "unmetered"); let vod = session("on_demand_hidden"); assert_eq!((vod["ended"].as_str(), vod["presentation"].as_str(), vod["attachments"].as_u64()), (Some("hidden"), Some("on_demand"), Some(1))); assert_eq!(vod["energy"][0]["class"], "OnChipCounter"); let none = session("nothing_presented"); assert_eq!((none["ended"].as_str(), none["presented_ms"].as_u64()), (Some("error"), Some(0))); assert_eq!(none["startup_ms"], serde_json::Value::Null); let cv = session("claim_verdicts"); let verdicts: BTreeSet<&str> = cv["claims"].as_array().unwrap().iter().map(|c| c["verdict"].as_str().unwrap()).collect(); for v in ["presented", "claim_malformed", "bad_key", "bad_signature", "audit_claim_unbound", "audit_marker_mismatch", "claim_duplicate", "claim_asset_mismatch", "audit_ad_not_acknowledged"] { assert!(verdicts.contains(v), "no claim of claim_verdicts is {v}"); } let report = |n: &str| doc["reports"].as_array().unwrap().iter().find(|v| v["name"] == n).unwrap_or_else(|| panic!("no report {n}")).clone(); let news = report("news_east"); assert_eq!(news["outcome"]["outcome"], "accept"); assert_eq!(news["served"]["report"]["rebuffering"]["suppressed"], true, "fewer than 50 sessions"); let fifty = report("fifty"); assert_eq!(fifty["outcome"]["outcome"], "accept"); let reb = &fifty["served"]["report"]["rebuffering"]; assert_eq!((reb["suppressed"].as_bool(), reb["sessions"].as_u64()), (Some(false), Some(52))); assert!(reb["stalls"].as_u64().unwrap() > 0); let energy = fifty["served"]["report"]["energy"].as_array().unwrap(); assert!(energy.iter().all(|e| e["class"] != "unmetered"), "the unmetered marker is never a total"); assert!(fifty["served"]["report"]["unmetered"]["sink_decode"].as_u64().unwrap() > 0); assert!(fifty["served"]["report"]["absent"]["display"].as_u64().unwrap() > 0); } /// §14 of the extension lists exactly the codes `Reason::ALL` has, each once, in order. #[test] fn the_extension_lists_the_reasons() { let text = read("extensions/stream-audit.md"); let at = text.find("\n## 14. Reasons").expect("a §14"); let end = text[at + 1..].find("\n## ").map_or(text.len(), |e| at + 1 + e); let listed: Vec = text[at..end].lines().filter(|l| l.starts_with("| `")).filter_map(|l| l.split('`').nth(1).map(String::from)).collect(); let want: Vec = Reason::ALL.iter().map(|r| r.code().to_string()).collect(); assert_eq!(listed, want, "§14's table and Reason::ALL differ"); }