feat: WASM Spec Testsuite (Attemp 2)

Signed-off-by: George Cosma <[email protected]>

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/specification/testsuite"]
+	path = tests/specification/testsuite
+	url = https://github.com/WebAssembly/testsuite.git

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "rust-analyzer.cargo.features": [
+        "spec-test"
+    ]
+}

Cargo.toml

@@ -21,13 +21,13 @@ env_logger = "0.10.1"
 wasmparser = "0.119.0"
 itertools = "0.12.0"
 wat = "1.0.83"
+wast = "212.0.0"
 criterion = { version = "0.5.1", features = ["html_reports"] }
 
-
 [features]
 default = ["hooks"]
 hooks = []
-
+spec-test = []
 
 [[bench]]
 name = "hook_performance_impact"

src/execution/mod.rs

@@ -93,6 +93,31 @@ where
         }
     }
 
+    pub fn invoke_named_dynamic(
+        &mut self,
+        func_name: &str,
+        param: Vec<Value>,
+        ret_types: &[ValType],
+    ) -> Result<Vec<Value>> {
+        // TODO: Optimize this search for better than linear-time. Pre-processing will likely be required
+        let func_idx = self.exports.iter().find_map(|export| {
+            if export.name == func_name {
+                match export.desc {
+                    ExportDesc::FuncIdx(idx) => Some(idx),
+                    _ => None,
+                }
+            } else {
+                None
+            }
+        });
+
+        if let Some(func_idx) = func_idx {
+            self.invoke_dynamic(func_idx, param, ret_types)
+        } else {
+            Err(RuntimeError(FunctionNotFound))
+        }
+    }
+
     /// Can only invoke functions with signature `[t1] -> [t2]` as of now.
     pub fn invoke_func<Param: InteropValueList, Returns: InteropValueList>(
         &mut self,

tests/specification/dummy.wast

@@ -0,0 +1,95 @@
+;; i32 operations
+
+(module
+  (func (export "add") (param $x i32) (param $y i32) (result i32) (i32.add (local.get $x) (local.get $y)))
+  (func (export "mul") (param $x i32) (param $y i32) (result i32) (i32.mul (local.get $x) (local.get $y)))
+  (func (export "div_s") (param $x i32) (param $y i32) (result i32) (i32.div_s (local.get $x) (local.get $y)))
+  (func (export "div_u") (param $x i32) (param $y i32) (result i32) (i32.div_u (local.get $x) (local.get $y)))
+  (func (export "rem_s") (param $x i32) (param $y i32) (result i32) (i32.rem_s (local.get $x) (local.get $y)))
+  (func (export "rem_u") (param $x i32) (param $y i32) (result i32) (i32.rem_u (local.get $x) (local.get $y)))
+)
+
+(assert_return (invoke "add" (i32.const 1) (i32.const 1)) (i32.const 2))
+(assert_return (invoke "add" (i32.const 1) (i32.const 0)) (i32.const 1))
+(assert_return (invoke "add" (i32.const -1) (i32.const -1)) (i32.const -2))
+(assert_return (invoke "add" (i32.const -1) (i32.const 1)) (i32.const 0))
+(assert_return (invoke "add" (i32.const 0x7fffffff) (i32.const 1)) (i32.const 0x80000000))
+(assert_return (invoke "add" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x7fffffff))
+(assert_return (invoke "add" (i32.const 0x80000000) (i32.const 0x80000000)) (i32.const 0))
+(assert_return (invoke "add" (i32.const 0x3fffffff) (i32.const 1)) (i32.const 0x40000000))
+
+(assert_return (invoke "mul" (i32.const 1) (i32.const 1)) (i32.const 1))
+(assert_return (invoke "mul" (i32.const 1) (i32.const 0)) (i32.const 0))
+(assert_return (invoke "mul" (i32.const -1) (i32.const -1)) (i32.const 1))
+(assert_return (invoke "mul" (i32.const 0x10000000) (i32.const 4096)) (i32.const 0))
+(assert_return (invoke "mul" (i32.const 0x80000000) (i32.const 0)) (i32.const 0))
+(assert_return (invoke "mul" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x80000000))
+(assert_return (invoke "mul" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0x80000001))
+(assert_return (invoke "mul" (i32.const 0x01234567) (i32.const 0x76543210)) (i32.const 0x358e7470))
+(assert_return (invoke "mul" (i32.const 0x7fffffff) (i32.const 0x7fffffff)) (i32.const 1))
+
+(assert_return (invoke "div_s" (i32.const 1) (i32.const 1)) (i32.const 1))
+(assert_return (invoke "div_s" (i32.const 0) (i32.const 1)) (i32.const 0))
+(assert_return (invoke "div_s" (i32.const 0) (i32.const -1)) (i32.const 0))
+(assert_return (invoke "div_s" (i32.const -1) (i32.const -1)) (i32.const 1))
+(assert_return (invoke "div_s" (i32.const 0x80000000) (i32.const 2)) (i32.const 0xc0000000))
+(assert_return (invoke "div_s" (i32.const 0x80000001) (i32.const 1000)) (i32.const 0xffdf3b65))
+(assert_return (invoke "div_s" (i32.const 5) (i32.const 2)) (i32.const 2))
+(assert_return (invoke "div_s" (i32.const -5) (i32.const 2)) (i32.const -2))
+(assert_return (invoke "div_s" (i32.const 5) (i32.const -2)) (i32.const -2))
+(assert_return (invoke "div_s" (i32.const -5) (i32.const -2)) (i32.const 2))
+(assert_return (invoke "div_s" (i32.const 7) (i32.const 3)) (i32.const 2))
+(assert_return (invoke "div_s" (i32.const -7) (i32.const 3)) (i32.const -2))
+(assert_return (invoke "div_s" (i32.const 7) (i32.const -3)) (i32.const -2))
+(assert_return (invoke "div_s" (i32.const -7) (i32.const -3)) (i32.const 2))
+(assert_return (invoke "div_s" (i32.const 11) (i32.const 5)) (i32.const 2))
+(assert_return (invoke "div_s" (i32.const 17) (i32.const 7)) (i32.const 2))
+
+(assert_return (invoke "div_u" (i32.const 1) (i32.const 1)) (i32.const 1))
+(assert_return (invoke "div_u" (i32.const 0) (i32.const 1)) (i32.const 0))
+(assert_return (invoke "div_u" (i32.const -1) (i32.const -1)) (i32.const 1))
+(assert_return (invoke "div_u" (i32.const 0x80000000) (i32.const -1)) (i32.const 0))
+(assert_return (invoke "div_u" (i32.const 0x80000000) (i32.const 2)) (i32.const 0x40000000))
+(assert_return (invoke "div_u" (i32.const 0x8ff00ff0) (i32.const 0x10001)) (i32.const 0x8fef))
+(assert_return (invoke "div_u" (i32.const 0x80000001) (i32.const 1000)) (i32.const 0x20c49b))
+(assert_return (invoke "div_u" (i32.const 5) (i32.const 2)) (i32.const 2))
+(assert_return (invoke "div_u" (i32.const -5) (i32.const 2)) (i32.const 0x7ffffffd))
+(assert_return (invoke "div_u" (i32.const 5) (i32.const -2)) (i32.const 0))
+(assert_return (invoke "div_u" (i32.const -5) (i32.const -2)) (i32.const 0))
+(assert_return (invoke "div_u" (i32.const 7) (i32.const 3)) (i32.const 2))
+(assert_return (invoke "div_u" (i32.const 11) (i32.const 5)) (i32.const 2))
+(assert_return (invoke "div_u" (i32.const 17) (i32.const 7)) (i32.const 2))
+
+(assert_return (invoke "rem_s" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0))
+(assert_return (invoke "rem_s" (i32.const 1) (i32.const 1)) (i32.const 0))
+(assert_return (invoke "rem_s" (i32.const 0) (i32.const 1)) (i32.const 0))
+(assert_return (invoke "rem_s" (i32.const 0) (i32.const -1)) (i32.const 0))
+(assert_return (invoke "rem_s" (i32.const -1) (i32.const -1)) (i32.const 0))
+(assert_return (invoke "rem_s" (i32.const 0x80000000) (i32.const -1)) (i32.const 0))
+(assert_return (invoke "rem_s" (i32.const 0x80000000) (i32.const 2)) (i32.const 0))
+(assert_return (invoke "rem_s" (i32.const 0x80000001) (i32.const 1000)) (i32.const -647))
+(assert_return (invoke "rem_s" (i32.const 5) (i32.const 2)) (i32.const 1))
+(assert_return (invoke "rem_s" (i32.const -5) (i32.const 2)) (i32.const -1))
+(assert_return (invoke "rem_s" (i32.const 5) (i32.const -2)) (i32.const 1))
+(assert_return (invoke "rem_s" (i32.const -5) (i32.const -2)) (i32.const -1))
+(assert_return (invoke "rem_s" (i32.const 7) (i32.const 3)) (i32.const 1))
+(assert_return (invoke "rem_s" (i32.const -7) (i32.const 3)) (i32.const -1))
+(assert_return (invoke "rem_s" (i32.const 7) (i32.const -3)) (i32.const 1))
+(assert_return (invoke "rem_s" (i32.const -7) (i32.const -3)) (i32.const -1))
+(assert_return (invoke "rem_s" (i32.const 11) (i32.const 5)) (i32.const 1))
+(assert_return (invoke "rem_s" (i32.const 17) (i32.const 7)) (i32.const 3))
+
+(assert_return (invoke "rem_u" (i32.const 1) (i32.const 1)) (i32.const 0))
+(assert_return (invoke "rem_u" (i32.const 0) (i32.const 1)) (i32.const 0))
+(assert_return (invoke "rem_u" (i32.const -1) (i32.const -1)) (i32.const 0))
+(assert_return (invoke "rem_u" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x80000000))
+(assert_return (invoke "rem_u" (i32.const 0x80000000) (i32.const 2)) (i32.const 0))
+(assert_return (invoke "rem_u" (i32.const 0x8ff00ff0) (i32.const 0x10001)) (i32.const 0x8001))
+(assert_return (invoke "rem_u" (i32.const 0x80000001) (i32.const 1000)) (i32.const 649))
+(assert_return (invoke "rem_u" (i32.const 5) (i32.const 2)) (i32.const 1))
+(assert_return (invoke "rem_u" (i32.const -5) (i32.const 2)) (i32.const 1))
+(assert_return (invoke "rem_u" (i32.const 5) (i32.const -2)) (i32.const 5))
+(assert_return (invoke "rem_u" (i32.const -5) (i32.const -2)) (i32.const -5))
+(assert_return (invoke "rem_u" (i32.const 7) (i32.const 3)) (i32.const 1))
+(assert_return (invoke "rem_u" (i32.const 11) (i32.const 5)) (i32.const 1))
+(assert_return (invoke "rem_u" (i32.const 17) (i32.const 7)) (i32.const 3))

tests/specification/mod.rs

@@ -0,0 +1,47 @@
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
+
+mod reports;
+mod run;
+mod test_errors;
+
+#[test_log::test]
+pub fn spec_dummy() {
+    let path = "./tests/specification/dummy.wast";
+    let report = run::run_spec_test(path);
+    println!("Report for {}:\n{}", path, report);
+}
+
+#[test_log::test]
+pub fn spec_tests() {
+    let paths = get_wast_files(Path::new("./tests/specification/testsuite/"))
+        .expect("Failed to find testsuite");
+    for test_path in paths {
+        let report = run::run_spec_test(test_path.to_str().unwrap());
+        println!("Report for {}:\n{}", test_path.display(), report);
+    }
+}
+
+// See: https://stackoverflow.com/a/76820878
+fn get_wast_files(base_path: &Path) -> Result<Vec<PathBuf>, std::io::Error> {
+    let mut buf = vec![];
+    let entries = fs::read_dir(base_path)?;
+
+    for entry in entries {
+        let entry = entry?;
+        let meta = entry.metadata()?;
+
+        if meta.is_dir() {
+            let mut subdir = get_wast_files(&entry.path())?;
+            buf.append(&mut subdir);
+        }
+
+        if meta.is_file() && entry.path().extension().unwrap_or_default() == "wast" {
+            buf.push(entry.path());
+        }
+    }
+
+    Ok(buf)
+}

tests/specification/reports.rs

@@ -0,0 +1,126 @@
+use std::error::Error;
+
+pub struct WastSuccess {
+    filename: String,
+    line_number: u32,
+    command: String,
+}
+
+pub struct WastError {
+    inner: Box<dyn Error>,
+    filename: String,
+    line_number: u32,
+    command: String,
+}
+
+impl WastError {
+    pub fn from_outside(error: Box<dyn Error>, reason: &str) -> Self {
+        Self {
+            inner: error,
+            filename: String::from(""),
+            line_number: 0,
+            command: String::from(reason),
+        }
+    }
+}
+
+pub struct AssertReport {
+    results: Vec<Result<WastSuccess, WastError>>,
+}
+
+impl AssertReport {
+    pub fn new() -> Self {
+        Self {
+            results: Vec::new(),
+        }
+    }
+
+    pub fn push_success(&mut self, filename: String, line_number: u32, command: String) {
+        self.results.push(Ok(WastSuccess {
+            filename,
+            line_number,
+            command,
+        }));
+    }
+
+    pub fn push_error(
+        &mut self,
+        filename: String,
+        line_number: u32,
+        command: String,
+        error: Box<dyn Error>,
+    ) {
+        self.results.push(Err(WastError {
+            inner: error,
+            filename,
+            line_number,
+            command,
+        }));
+    }
+}
+
+pub enum WastTestReport {
+    Asserts(AssertReport),
+    CompilationError(WastError),
+}
+
+impl From<WastError> for WastTestReport {
+    fn from(error: WastError) -> Self {
+        WastTestReport::CompilationError(error)
+    }
+}
+
+impl From<AssertReport> for WastTestReport {
+    fn from(assert_report: AssertReport) -> Self {
+        WastTestReport::Asserts(assert_report)
+    }
+}
+
+// .------------------------.
+// | Display Implementation |
+// '------------------------'
+
+impl std::fmt::Display for WastSuccess {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "[SUCCESS] {} ({}:{})",
+            self.command, self.filename, self.line_number
+        )
+    }
+}
+
+impl std::fmt::Display for WastError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(
+            f,
+            "[ERROR] {} ({}:{})",
+            self.command, self.filename, self.line_number
+        )?;
+        write!(f, "\t{}", self.inner)?;
+
+        Ok(())
+    }
+}
+
+impl std::fmt::Display for AssertReport {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for result in &self.results {
+            match result {
+                Ok(success) => writeln!(f, "{}", success)?,
+                Err(error) => writeln!(f, "{}", error)?,
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl std::fmt::Display for WastTestReport {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            WastTestReport::Asserts(assert_report) => write!(f, "{}", assert_report),
+            WastTestReport::CompilationError(error) => write!(f, "{}", error),
+        }
+    }
+}