open-standards/wai/wai-module-check/examples/spectest.rs

//! The WebAssembly specification's tests as a third corpus for the differential.
//!
//!     cargo run --release --example spectest -- --wast DIR --out DIR
//!
//! Reads every `.wast` file under `--wast` (the specification's `test/core`), and writes
//! each module its scripts define, assert invalid or assert malformed, in binary, to
//! `--out` as `<file>.<n>.wasm`, with this checker's verdicts in `verdicts.txt` in the form
//! differential.mjs reads (profile caps, both levels). A module given only as text that
//! does not parse is a test of the text format and is left out, as is a component.
//!
//! wasm-smith's modules test how two validators accept far more than how they refuse; the
//! specification's `assert_invalid` and `assert_malformed` modules are the refusals,
//! one construct at a time. It prints, for each of the script's expectations, how many of
//! its modules this checker refused by R2: a valid module refused is one that uses a
//! construct outside WebAssembly 3.0 or a part beyond a validation limit (§4.2), and an
//! invalid one not refused would be this validator's error. Either, or a script it cannot
//! read, is exit status 1.

use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::path::{Path, PathBuf};

use wai_module_check::{check, Caps, Level, Verdict};
use wast::parser::{self, ParseBuffer};
use wast::{QuoteWat, Wast, WastDirective, Wat};

fn wast_files(dir: &Path, out: &mut Vec<PathBuf>) {
    let mut entries: Vec<_> = std::fs::read_dir(dir).expect("the test directory").flatten().map(|e| e.path()).collect();
    entries.sort();
    for p in entries {
        if p.is_dir() {
            wast_files(&p, out);
        } else if p.extension().is_some_and(|e| e == "wast") {
            out.push(p);
        }
    }
}

/// Writes every module of one script; `None` where the script does not parse.
fn parse_script(
    buf: &ParseBuffer<'_>,
    stem: &str,
    out: &Path,
    caps: &Caps,
    lines: &mut String,
    tally: &mut BTreeMap<(&'static str, bool), u64>,
) -> Option<()> {
    let script = parser::parse::<Wast>(buf).ok()?;
    for (n, directive) in script.directives.into_iter().enumerate() {
        let (mut module, expect) = match directive {
            WastDirective::Module(m) | WastDirective::ModuleDefinition(m) => (m, "valid"),
            WastDirective::AssertInvalid { module, .. } => (module, "invalid"),
            WastDirective::AssertMalformed { module, .. } => (module, "malformed"),
            _ => continue,
        };
        if matches!(module, QuoteWat::QuoteComponent(..) | QuoteWat::Wat(Wat::Component(_))) {
            continue;
        }
        let Ok(bytes) = module.encode() else { continue };
        let name = format!("{stem}.{n}");
        std::fs::write(out.join(format!("{name}.wasm")), &bytes).expect("a module");
        let int = check(&bytes, Level::Integer, caps);
        let float = check(&bytes, Level::Float, caps);
        writeln!(lines, "{name} - {int} | {float}").unwrap();
        let r2 = matches!(int, Verdict::Refused { rule: 2, .. });
        *tally.entry((expect, r2)).or_insert(0u64) += 1;
    }
    Some(())
}

fn main() {
    let (mut wast_dir, mut out) = (None, None);
    let mut args = std::env::args().skip(1);
    while let Some(a) = args.next() {
        match a.as_str() {
            "--wast" => wast_dir = args.next().map(PathBuf::from),
            "--out" => out = args.next().map(PathBuf::from),
            _ => panic!("usage: spectest --wast DIR --out DIR"),
        }
    }
    let (wast_dir, out) = (wast_dir.expect("--wast DIR"), out.expect("--out DIR"));
    std::fs::create_dir_all(&out).expect("the output directory");
    let mut files = Vec::new();
    wast_files(&wast_dir, &mut files);
    let caps = Caps::PROFILE;
    let (mut lines, mut tally, mut unread) = (String::new(), BTreeMap::new(), 0);
    for file in &files {
        let stem = file
            .strip_prefix(&wast_dir)
            .unwrap_or(file)
            .with_extension("")
            .to_string_lossy()
            .replace(['/', '\\'], "_");
        let parsed = std::fs::read_to_string(file).ok().and_then(|text| {
            let buf = ParseBuffer::new(&text).ok()?;
            parse_script(&buf, &stem, &out, &caps, &mut lines, &mut tally)
        });
        if parsed.is_none() {
            println!("  not read: {}", file.display());
            // names.wast spells export names with bidirectional controls, which the text
            // parser refuses by design. It holds no invalid or malformed module.
            if !file.ends_with("names.wast") {
                unread += 1;
            }
        }
    }
    std::fs::write(out.join("verdicts.txt"), lines).expect("verdicts.txt");
    for ((expect, r2), n) in &tally {
        println!("  {expect}, {}: {n}", if *r2 { "refused R2" } else { "not refused R2" });
    }
    println!(
        "spectest: {} modules from {} scripts in {} ({unread} scripts this parser does not read)",
        tally.values().sum::<u64>(),
        files.len(),
        out.display()
    );
    // A valid module is held to R2 and passes it; an invalid or malformed one is refused by it.
    let wrong: u64 = tally.iter().filter(|((expect, r2), _)| (*expect == "valid") == *r2).map(|(_, n)| *n).sum();
    if wrong != 0 || unread != 0 {
        eprintln!("spectest: {wrong} modules against their script's expectation, {unread} scripts not read");
        std::process::exit(1);
    }
}