From 76d9b4282e5b7f78afc25859a6ebd700eb2f14a6 Mon Sep 17 00:00:00 2001 From: George Cosma Date: Fri, 26 Jul 2024 13:25:45 +0300 Subject: [PATCH 1/2] feat: WASM Spec Testsuite (Attempt 2) Signed-off-by: George Cosma --- .gitmodules | 3 + Cargo.lock | 27 ++- Cargo.toml | 2 + src/execution/mod.rs | 25 +++ tests/specification/dummy.wast | 95 ++++++++++ tests/specification/mod.rs | 47 +++++ tests/specification/reports.rs | 135 ++++++++++++++ tests/specification/run.rs | 280 +++++++++++++++++++++++++++++ tests/specification/test_errors.rs | 88 +++++++++ tests/specification/testsuite | 1 + tests/wasm_spec_testsuite.rs | 5 + 11 files changed, 706 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 tests/specification/dummy.wast create mode 100644 tests/specification/mod.rs create mode 100644 tests/specification/reports.rs create mode 100644 tests/specification/run.rs create mode 100644 tests/specification/test_errors.rs create mode 160000 tests/specification/testsuite create mode 100644 tests/wasm_spec_testsuite.rs diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..993e1db1 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests/specification/testsuite"] + path = tests/specification/testsuite + url = https://github.com/WebAssembly/testsuite.git diff --git a/Cargo.lock b/Cargo.lock index 02a96a65..3f58ba2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -659,6 +659,15 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-encoder" +version = "0.212.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501940df4418b8929eb6d52f1aade1fdd15a5b86c92453cb696e3c906bd3fc33" +dependencies = [ + "leb128", +] + [[package]] name = "wasm-interpreter" version = "0.1.0" @@ -671,6 +680,7 @@ dependencies = [ "log", "test-log", "wasmparser", + "wast 212.0.0", "wat", ] @@ -695,7 +705,20 @@ dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder", + "wasm-encoder 0.200.0", +] + +[[package]] +name = "wast" +version = "212.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4606a05fb0aae5d11dd7d8280a640d88a63ee019360ba9be552da3d294b8d1f5" +dependencies = [ + "bumpalo", + "leb128", + "memchr", + "unicode-width", + "wasm-encoder 0.212.0", ] [[package]] @@ -704,7 +727,7 @@ version = "1.200.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "776cbd10e217f83869beaa3f40e312bb9e91d5eee29bbf6f560db1261b6a4c3d" dependencies = [ - "wast", + "wast 200.0.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c41468e0..54ed3ec9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,12 +28,14 @@ env_logger = "0.10.1" wasmparser = "0.119.0" itertools = "0.12.0" wat = "1.0.83" +wast = "212.0.0" criterion = { version = "0.5.1", features = ["html_reports"] } hexf = "0.2.1" [features] default = ["hooks"] hooks = [] +spec-test = [] [[bench]] name = "hook_performance_impact" diff --git a/src/execution/mod.rs b/src/execution/mod.rs index 07a2f500..52155e34 100644 --- a/src/execution/mod.rs +++ b/src/execution/mod.rs @@ -96,6 +96,31 @@ where } } + pub fn invoke_named_dynamic( + &mut self, + func_name: &str, + param: Vec, + ret_types: &[ValType], + ) -> Result, RuntimeError> { + // TODO: Optimize this search for better than linear-time. Pre-processing will likely be required + let func_idx = self.exports.iter().find_map(|export| { + if export.name == func_name { + match export.desc { + ExportDesc::FuncIdx(idx) => Some(idx), + _ => None, + } + } else { + None + } + }); + + if let Some(func_idx) = func_idx { + self.invoke_dynamic(func_idx, param, ret_types) + } else { + Err(RuntimeError::FunctionNotFound) + } + } + /// Can only invoke functions with signature `[t1] -> [t2]` as of now. pub fn invoke_func( &mut self, diff --git a/tests/specification/dummy.wast b/tests/specification/dummy.wast new file mode 100644 index 00000000..792d949e --- /dev/null +++ b/tests/specification/dummy.wast @@ -0,0 +1,95 @@ +;; i32 operations + +(module + (func (export "add") (param $x i32) (param $y i32) (result i32) (i32.add (local.get $x) (local.get $y))) + (func (export "mul") (param $x i32) (param $y i32) (result i32) (i32.mul (local.get $x) (local.get $y))) + (func (export "div_s") (param $x i32) (param $y i32) (result i32) (i32.div_s (local.get $x) (local.get $y))) + (func (export "div_u") (param $x i32) (param $y i32) (result i32) (i32.div_u (local.get $x) (local.get $y))) + (func (export "rem_s") (param $x i32) (param $y i32) (result i32) (i32.rem_s (local.get $x) (local.get $y))) + (func (export "rem_u") (param $x i32) (param $y i32) (result i32) (i32.rem_u (local.get $x) (local.get $y))) +) + +(assert_return (invoke "add" (i32.const 1) (i32.const 1)) (i32.const 2)) +(assert_return (invoke "add" (i32.const 1) (i32.const 0)) (i32.const 1)) +(assert_return (invoke "add" (i32.const -1) (i32.const -1)) (i32.const -2)) +(assert_return (invoke "add" (i32.const -1) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "add" (i32.const 0x7fffffff) (i32.const 1)) (i32.const 0x80000000)) +(assert_return (invoke "add" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x7fffffff)) +(assert_return (invoke "add" (i32.const 0x80000000) (i32.const 0x80000000)) (i32.const 0)) +(assert_return (invoke "add" (i32.const 0x3fffffff) (i32.const 1)) (i32.const 0x40000000)) + +(assert_return (invoke "mul" (i32.const 1) (i32.const 1)) (i32.const 1)) +(assert_return (invoke "mul" (i32.const 1) (i32.const 0)) (i32.const 0)) +(assert_return (invoke "mul" (i32.const -1) (i32.const -1)) (i32.const 1)) +(assert_return (invoke "mul" (i32.const 0x10000000) (i32.const 4096)) (i32.const 0)) +(assert_return (invoke "mul" (i32.const 0x80000000) (i32.const 0)) (i32.const 0)) +(assert_return (invoke "mul" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x80000000)) +(assert_return (invoke "mul" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0x80000001)) +(assert_return (invoke "mul" (i32.const 0x01234567) (i32.const 0x76543210)) (i32.const 0x358e7470)) +(assert_return (invoke "mul" (i32.const 0x7fffffff) (i32.const 0x7fffffff)) (i32.const 1)) + +(assert_return (invoke "div_s" (i32.const 1) (i32.const 1)) (i32.const 1)) +(assert_return (invoke "div_s" (i32.const 0) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "div_s" (i32.const 0) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "div_s" (i32.const -1) (i32.const -1)) (i32.const 1)) +(assert_return (invoke "div_s" (i32.const 0x80000000) (i32.const 2)) (i32.const 0xc0000000)) +(assert_return (invoke "div_s" (i32.const 0x80000001) (i32.const 1000)) (i32.const 0xffdf3b65)) +(assert_return (invoke "div_s" (i32.const 5) (i32.const 2)) (i32.const 2)) +(assert_return (invoke "div_s" (i32.const -5) (i32.const 2)) (i32.const -2)) +(assert_return (invoke "div_s" (i32.const 5) (i32.const -2)) (i32.const -2)) +(assert_return (invoke "div_s" (i32.const -5) (i32.const -2)) (i32.const 2)) +(assert_return (invoke "div_s" (i32.const 7) (i32.const 3)) (i32.const 2)) +(assert_return (invoke "div_s" (i32.const -7) (i32.const 3)) (i32.const -2)) +(assert_return (invoke "div_s" (i32.const 7) (i32.const -3)) (i32.const -2)) +(assert_return (invoke "div_s" (i32.const -7) (i32.const -3)) (i32.const 2)) +(assert_return (invoke "div_s" (i32.const 11) (i32.const 5)) (i32.const 2)) +(assert_return (invoke "div_s" (i32.const 17) (i32.const 7)) (i32.const 2)) + +(assert_return (invoke "div_u" (i32.const 1) (i32.const 1)) (i32.const 1)) +(assert_return (invoke "div_u" (i32.const 0) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "div_u" (i32.const -1) (i32.const -1)) (i32.const 1)) +(assert_return (invoke "div_u" (i32.const 0x80000000) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "div_u" (i32.const 0x80000000) (i32.const 2)) (i32.const 0x40000000)) +(assert_return (invoke "div_u" (i32.const 0x8ff00ff0) (i32.const 0x10001)) (i32.const 0x8fef)) +(assert_return (invoke "div_u" (i32.const 0x80000001) (i32.const 1000)) (i32.const 0x20c49b)) +(assert_return (invoke "div_u" (i32.const 5) (i32.const 2)) (i32.const 2)) +(assert_return (invoke "div_u" (i32.const -5) (i32.const 2)) (i32.const 0x7ffffffd)) +(assert_return (invoke "div_u" (i32.const 5) (i32.const -2)) (i32.const 0)) +(assert_return (invoke "div_u" (i32.const -5) (i32.const -2)) (i32.const 0)) +(assert_return (invoke "div_u" (i32.const 7) (i32.const 3)) (i32.const 2)) +(assert_return (invoke "div_u" (i32.const 11) (i32.const 5)) (i32.const 2)) +(assert_return (invoke "div_u" (i32.const 17) (i32.const 7)) (i32.const 2)) + +(assert_return (invoke "rem_s" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "rem_s" (i32.const 1) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "rem_s" (i32.const 0) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "rem_s" (i32.const 0) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "rem_s" (i32.const -1) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "rem_s" (i32.const 0x80000000) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "rem_s" (i32.const 0x80000000) (i32.const 2)) (i32.const 0)) +(assert_return (invoke "rem_s" (i32.const 0x80000001) (i32.const 1000)) (i32.const -647)) +(assert_return (invoke "rem_s" (i32.const 5) (i32.const 2)) (i32.const 1)) +(assert_return (invoke "rem_s" (i32.const -5) (i32.const 2)) (i32.const -1)) +(assert_return (invoke "rem_s" (i32.const 5) (i32.const -2)) (i32.const 1)) +(assert_return (invoke "rem_s" (i32.const -5) (i32.const -2)) (i32.const -1)) +(assert_return (invoke "rem_s" (i32.const 7) (i32.const 3)) (i32.const 1)) +(assert_return (invoke "rem_s" (i32.const -7) (i32.const 3)) (i32.const -1)) +(assert_return (invoke "rem_s" (i32.const 7) (i32.const -3)) (i32.const 1)) +(assert_return (invoke "rem_s" (i32.const -7) (i32.const -3)) (i32.const -1)) +(assert_return (invoke "rem_s" (i32.const 11) (i32.const 5)) (i32.const 1)) +(assert_return (invoke "rem_s" (i32.const 17) (i32.const 7)) (i32.const 3)) + +(assert_return (invoke "rem_u" (i32.const 1) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "rem_u" (i32.const 0) (i32.const 1)) (i32.const 0)) +(assert_return (invoke "rem_u" (i32.const -1) (i32.const -1)) (i32.const 0)) +(assert_return (invoke "rem_u" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x80000000)) +(assert_return (invoke "rem_u" (i32.const 0x80000000) (i32.const 2)) (i32.const 0)) +(assert_return (invoke "rem_u" (i32.const 0x8ff00ff0) (i32.const 0x10001)) (i32.const 0x8001)) +(assert_return (invoke "rem_u" (i32.const 0x80000001) (i32.const 1000)) (i32.const 649)) +(assert_return (invoke "rem_u" (i32.const 5) (i32.const 2)) (i32.const 1)) +(assert_return (invoke "rem_u" (i32.const -5) (i32.const 2)) (i32.const 1)) +(assert_return (invoke "rem_u" (i32.const 5) (i32.const -2)) (i32.const 5)) +(assert_return (invoke "rem_u" (i32.const -5) (i32.const -2)) (i32.const -5)) +(assert_return (invoke "rem_u" (i32.const 7) (i32.const 3)) (i32.const 1)) +(assert_return (invoke "rem_u" (i32.const 11) (i32.const 5)) (i32.const 1)) +(assert_return (invoke "rem_u" (i32.const 17) (i32.const 7)) (i32.const 3)) diff --git a/tests/specification/mod.rs b/tests/specification/mod.rs new file mode 100644 index 00000000..65cc75e7 --- /dev/null +++ b/tests/specification/mod.rs @@ -0,0 +1,47 @@ +use std::{ + fs, + path::{Path, PathBuf}, +}; + +mod reports; +mod run; +mod test_errors; + +#[test_log::test] +pub fn spec_dummy() { + let path = "./tests/specification/dummy.wast"; + let report = run::run_spec_test(path); + println!("Report for {}:\n{}", path, report); +} + +#[test_log::test] +pub fn spec_tests() { + let paths = get_wast_files(Path::new("./tests/specification/testsuite/")) + .expect("Failed to find testsuite"); + for test_path in paths { + let report = run::run_spec_test(test_path.to_str().unwrap()); + println!("Report for {}:\n{}", test_path.display(), report); + } +} + +// See: https://stackoverflow.com/a/76820878 +fn get_wast_files(base_path: &Path) -> Result, std::io::Error> { + let mut buf = vec![]; + let entries = fs::read_dir(base_path)?; + + for entry in entries { + let entry = entry?; + let meta = entry.metadata()?; + + if meta.is_dir() { + let mut subdir = get_wast_files(&entry.path())?; + buf.append(&mut subdir); + } + + if meta.is_file() && entry.path().extension().unwrap_or_default() == "wast" { + buf.push(entry.path()); + } + } + + Ok(buf) +} diff --git a/tests/specification/reports.rs b/tests/specification/reports.rs new file mode 100644 index 00000000..af6cfb32 --- /dev/null +++ b/tests/specification/reports.rs @@ -0,0 +1,135 @@ +use std::error::Error; + +pub struct WastSuccess { + filename: String, + line_number: u32, + command: String, +} + +pub struct WastError { + inner: Box, + filename: String, + line_number: u32, + command: String, +} + +impl WastError { + pub fn new(error: Box, filename: String, line_number: u32, command: &str) -> Self { + Self { + inner: error, + filename, + line_number, + command: command.to_string(), + } + } + + pub fn from_outside(error: Box, reason: &str) -> Self { + Self { + inner: error, + filename: "".to_string(), + line_number: 0, + command: reason.to_string(), + } + } +} + +pub struct AssertReport { + results: Vec>, +} + +impl AssertReport { + pub fn new() -> Self { + Self { + results: Vec::new(), + } + } + + pub fn push_success(&mut self, filename: String, line_number: u32, command: String) { + self.results.push(Ok(WastSuccess { + filename, + line_number, + command, + })); + } + + pub fn push_error( + &mut self, + filename: String, + line_number: u32, + command: String, + error: Box, + ) { + self.results.push(Err(WastError { + inner: error, + filename, + line_number, + command, + })); + } +} + +pub enum WastTestReport { + Asserts(AssertReport), + CompilationError(WastError), +} + +impl From for WastTestReport { + fn from(error: WastError) -> Self { + WastTestReport::CompilationError(error) + } +} + +impl From for WastTestReport { + fn from(assert_report: AssertReport) -> Self { + WastTestReport::Asserts(assert_report) + } +} + +// .------------------------. +// | Display Implementation | +// '------------------------' + +impl std::fmt::Display for WastSuccess { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "[SUCCESS] {} ({}:{})", + self.command, self.filename, self.line_number + ) + } +} + +impl std::fmt::Display for WastError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "[ERROR] {} ({}:{})", + self.command, self.filename, self.line_number + )?; + write!(f, "\t{}", self.inner)?; + + Ok(()) + } +} + +impl std::fmt::Display for AssertReport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for result in &self.results { + match result { + Ok(success) => writeln!(f, "{}", success)?, + Err(error) => writeln!(f, "{}", error)?, + } + } + + Ok(()) + } +} + +impl std::fmt::Display for WastTestReport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WastTestReport::Asserts(assert_report) => write!(f, "{}", assert_report), + WastTestReport::CompilationError(error) => write!(f, "{}", error), + } + } +} diff --git a/tests/specification/run.rs b/tests/specification/run.rs new file mode 100644 index 00000000..a856efd0 --- /dev/null +++ b/tests/specification/run.rs @@ -0,0 +1,280 @@ +use std::error::Error; +use std::panic::catch_unwind; +use std::panic::AssertUnwindSafe; + +use wasm::Value; +use wasm::{validate, RuntimeInstance}; + +use crate::specification::reports::*; +use crate::specification::test_errors::*; + +macro_rules! try_to { + ($e:expr) => { + match $e { + Ok(val) => val, + Err(err) => return err.into(), + } + }; +} + +pub fn run_spec_test(filepath: &str) -> WastTestReport { + // -=-= Initialization =-=- + let contents = try_to!(std::fs::read_to_string(filepath) + .map_err(|err| WastError::from_outside(Box::new(err), "failed to open wast file"))); + + let buf = try_to!(wast::parser::ParseBuffer::new(&contents) + .map_err(|err| WastError::from_outside(Box::new(err), "failed to create wast buffer"))); + + let wast = try_to!(wast::parser::parse::(&buf) + .map_err(|err| WastError::from_outside(Box::new(err), "failed to parse wast file"))); + + // -=-= Testing & Compilation =-=- + let mut asserts = AssertReport::new(); + + // We need to keep the wasm_bytes in-scope for the lifetime of the interpeter. + // As such, we hoist the bytes into an Option, and assign it once a module directive is found. + #[allow(unused_assignments)] + let mut wasm_bytes: Option> = None; + let mut interpeter = None; + + for directive in wast.directives { + match directive { + wast::WastDirective::Wat(mut quoted) => { + let encoded = try_to!(quoted + .encode() + .map_err(|err| WastError::from_outside(Box::new(err), "failed to encode wat"))); + + wasm_bytes = Some(encoded); + + let validation_attempt = catch_unwind(|| { + validate(wasm_bytes.as_ref().unwrap()).map_err(|err| { + WastError::new( + Box::::new(err.into()), + filepath.to_string(), + 0, + "Module validation failed", + ) + }) + }); + + let validation_info = match validation_attempt { + Ok(original_result) => try_to!(original_result), + Err(inner) => { + // TODO: Do we want to exit on panic? State may be left in an inconsistent state, and cascading panics may occur. + let err = if let Ok(msg) = inner.downcast::<&str>() { + Box::new(PanicError::new(msg.to_string())) + } else { + Box::new(PanicError::new("Unknown panic".into())) + }; + + return WastError::new( + err, + filepath.to_string(), + 0, + "Module validation panicked", + ) + .into(); + } + }; + + let instance = try_to!(RuntimeInstance::new(&validation_info).map_err(|err| { + let err: wasm::Error = err.into(); + let err: WasmInterpreterError = err.into(); + WastError::from_outside(Box::new(err), "failed to create runtime instance") + })); + + interpeter = Some(instance); + } + wast::WastDirective::AssertReturn { + span, + exec, + results, + } => { + if interpeter.is_none() { + asserts.push_error( + filepath.to_string(), + span.linecol_in(&contents).0 as u32 + 1, + get_command(&contents, span), + Box::new(GenericError::new( + "Attempted to run assert_return before a module was compiled", + )), + ); + continue; + } + + let interpeter = interpeter.as_mut().unwrap(); + + let err_or_panic: Result<(), Box> = + match catch_unwind(AssertUnwindSafe(|| { + execute_assert_return(interpeter, exec, results) + })) { + Ok(original_result) => original_result, + Err(inner) => { + // TODO: Do we want to exit on panic? State may be left in an inconsistent state, and cascading panics may occur. + if let Ok(msg) = inner.downcast::<&str>() { + Err(Box::new(PanicError::new(msg.to_string()))) + } else { + Err(Box::new(PanicError::new("Unknown panic".into()))) + } + } + }; + + match err_or_panic { + Ok(_) => { + asserts.push_success( + filepath.to_string(), + span.linecol_in(&contents).0 as u32 + 1, + get_command(&contents, span), + ); + } + Err(inner) => { + asserts.push_error( + filepath.to_string(), + span.linecol_in(&contents).0 as u32 + 1, + get_command(&contents, span), + inner, + ); + } + } + } + wast::WastDirective::AssertMalformed { + span, + module: _, + message: _, + } + | wast::WastDirective::AssertInvalid { + span, + module: _, + message: _, + } + | wast::WastDirective::Register { + span, + name: _, + module: _, + } + | wast::WastDirective::AssertTrap { + span, + exec: _, + message: _, + } + | wast::WastDirective::AssertExhaustion { + span, + call: _, + message: _, + } + | wast::WastDirective::AssertUnlinkable { + span, + module: _, + message: _, + } + | wast::WastDirective::AssertException { span, exec: _ } => { + asserts.push_error( + filepath.to_string(), + span.linecol_in(&contents).0 as u32 + 1, + get_command(&contents, span), + Box::new(GenericError::new("Assert directive not yet implemented")), + ); + } + wast::WastDirective::Wait { span: _, thread: _ } => todo!(), + wast::WastDirective::Invoke(_) => todo!(), + wast::WastDirective::Thread(_) => todo!(), + } + } + + asserts.into() +} + +fn execute_assert_return( + interpeter: &mut RuntimeInstance, + exec: wast::WastExecute, + results: Vec, +) -> Result<(), Box> { + match exec { + wast::WastExecute::Invoke(invoke_info) => { + let args = invoke_info + .args + .into_iter() + .map(arg_to_value) + .collect::>(); + + let result_vals = results.into_iter().map(result_to_value).collect::>(); + let result_types = result_vals + .iter() + .map(|val| val.to_ty()) + .collect::>(); + + let actual = interpeter + .invoke_named_dynamic(invoke_info.name, args, &result_types) + .map_err(|err| { + let err: wasm::Error = err.into(); + let err: WasmInterpreterError = err.into(); + Box::new(err) + })?; + + AssertEqError::assert_eq(actual, result_vals)?; + } + wast::WastExecute::Get { + span: _, + module: _, + global: _, + } => todo!(), + wast::WastExecute::Wat(_) => todo!(), + } + + Ok(()) +} + +pub fn arg_to_value(arg: wast::WastArg) -> Value { + match arg { + wast::WastArg::Core(core_arg) => match core_arg { + wast::core::WastArgCore::I32(val) => Value::I32(val as u32), + wast::core::WastArgCore::I64(val) => Value::I64(val as u64), + wast::core::WastArgCore::F32(val) => Value::F32(wasm::value::F32(val.bits as f32)), + wast::core::WastArgCore::F64(val) => Value::F64(wasm::value::F64(val.bits as f64)), + wast::core::WastArgCore::V128(_) => todo!(), + wast::core::WastArgCore::RefNull(_) => todo!(), + wast::core::WastArgCore::RefExtern(_) => todo!(), + wast::core::WastArgCore::RefHost(_) => todo!(), + }, + wast::WastArg::Component(_) => todo!(), + } +} + +fn result_to_value(result: wast::WastRet) -> Value { + match result { + wast::WastRet::Core(core_arg) => match core_arg { + wast::core::WastRetCore::I32(val) => Value::I32(val as u32), + wast::core::WastRetCore::I64(val) => Value::I64(val as u64), + wast::core::WastRetCore::F32(val) => match val { + wast::core::NanPattern::CanonicalNan => todo!(), + wast::core::NanPattern::ArithmeticNan => todo!(), + wast::core::NanPattern::Value(val) => Value::F32(wasm::value::F32(val.bits as f32)), + }, + wast::core::WastRetCore::F64(val) => match val { + wast::core::NanPattern::CanonicalNan => todo!(), + wast::core::NanPattern::ArithmeticNan => todo!(), + wast::core::NanPattern::Value(val) => Value::F64(wasm::value::F64(val.bits as f64)), + }, + wast::core::WastRetCore::V128(_) => todo!(), + wast::core::WastRetCore::RefNull(_) => todo!(), + wast::core::WastRetCore::RefExtern(_) => todo!(), + wast::core::WastRetCore::RefHost(_) => todo!(), + wast::core::WastRetCore::RefFunc(_) => todo!(), + wast::core::WastRetCore::RefAny => todo!(), + wast::core::WastRetCore::RefEq => todo!(), + wast::core::WastRetCore::RefArray => todo!(), + wast::core::WastRetCore::RefStruct => todo!(), + wast::core::WastRetCore::RefI31 => todo!(), + wast::core::WastRetCore::Either(_) => todo!(), + }, + wast::WastRet::Component(_) => todo!(), + } +} + +pub fn get_command(contents: &str, span: wast::token::Span) -> String { + contents[span.offset() as usize..] + .lines() + .next() + .unwrap_or("") + .to_string() +} diff --git a/tests/specification/test_errors.rs b/tests/specification/test_errors.rs new file mode 100644 index 00000000..0c94cfc7 --- /dev/null +++ b/tests/specification/test_errors.rs @@ -0,0 +1,88 @@ +use std::error::Error; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct AssertEqError { + left: String, + right: String, +} + +impl AssertEqError { + pub fn assert_eq(left: T, right: T) -> Result<(), Self> { + if left != right { + return Err(AssertEqError { + left: format!("{:?}", left), + right: format!("{:?}", right), + }); + } + + Ok(()) + } +} +impl Error for AssertEqError {} +impl std::fmt::Display for AssertEqError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "assert_eq failed: left: {}, right: {}", + self.left, self.right + ) + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct PanicError { + message: String, +} + +impl PanicError { + pub fn new(message: String) -> Self { + PanicError { message } + } +} + +impl Error for PanicError {} +impl std::fmt::Display for PanicError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Panic: {}", self.message) + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct WasmInterpreterError(wasm::Error); + +impl Error for WasmInterpreterError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match &self.0 { + wasm::Error::MalformedUtf8String(inner) => Some(inner), + _ => None, + } + } +} + +impl From for WasmInterpreterError { + fn from(error: wasm::Error) -> Self { + WasmInterpreterError(error) + } +} + +impl std::fmt::Display for WasmInterpreterError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "{}", self.0) + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct GenericError(String); + +impl GenericError { + pub fn new(message: &str) -> Self { + GenericError(message.to_string()) + } +} + +impl Error for GenericError {} +impl std::fmt::Display for GenericError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} diff --git a/tests/specification/testsuite b/tests/specification/testsuite new file mode 160000 index 00000000..53da17c0 --- /dev/null +++ b/tests/specification/testsuite @@ -0,0 +1 @@ +Subproject commit 53da17c0936a23f68f97cde4f9346a0a374dc35f diff --git a/tests/wasm_spec_testsuite.rs b/tests/wasm_spec_testsuite.rs new file mode 100644 index 00000000..0fdfb322 --- /dev/null +++ b/tests/wasm_spec_testsuite.rs @@ -0,0 +1,5 @@ +// The reason this file exists is only to expose the `specification` module to the outside world. +// More so, the reason it wasn't added to the `lib.rs` file is because we wanted to separate the +// regular tests from the spec tests. +#[cfg(feature = "spec-test")] +mod specification; From 3c8241417e7e6f842f4f017559edd97da0d0742d Mon Sep 17 00:00:00 2001 From: George Cosma Date: Mon, 2 Sep 2024 20:36:41 +0300 Subject: [PATCH 2/2] refactor: refactor spec testsuite and remove feature Signed-off-by: George Cosma --- .github/workflows/ci.yml | 6 + .github/workflows/nix.yml | 8 +- .github/workflows/pages_coverage_preview.yaml | 2 + .github/workflows/pages_deploy_main.yaml | 2 + .../workflows/pages_requirement_preview.yaml | 2 + Cargo.toml | 1 - pkgs/wasm-interpreter.nix | 2 +- tests/specification/dummy.wast | 95 ------- tests/specification/mod.rs | 33 ++- tests/specification/reports.rs | 172 ++++++------ tests/specification/run.rs | 246 +++++++++++------- tests/specification/test_errors.rs | 16 +- tests/specification/testsuite | 2 +- tests/wasm_spec_testsuite.rs | 1 - 14 files changed, 295 insertions(+), 293 deletions(-) delete mode 100644 tests/specification/dummy.wast diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62f8c901..ad6f312c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: true - name: Format run: cargo check - name: Run clippy @@ -39,11 +41,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: webiny/action-conventional-commits@v1.3.0 msrv: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: taiki-e/install-action@cargo-hack - run: cargo hack check --rust-version --workspace --all-targets --ignore-private diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index df57a3b0..f1fd174f 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} @@ -26,6 +28,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} @@ -33,7 +37,7 @@ jobs: with: name: dlr-ft authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - - run: nix build .#wasm-interpreter --print-build-logs + - run: nix build .?submodules=1#wasm-interpreter --print-build-logs - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: @@ -41,7 +45,7 @@ jobs: file: result/lcov-codecov.json env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - run: nix build .#report --print-build-logs + - run: nix build .?submodules=1#report --print-build-logs - name: Archive report uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/pages_coverage_preview.yaml b/.github/workflows/pages_coverage_preview.yaml index ed11443c..c22b0ce0 100644 --- a/.github/workflows/pages_coverage_preview.yaml +++ b/.github/workflows/pages_coverage_preview.yaml @@ -23,6 +23,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true # -=-=-=-= Create report =-=-=-=- - uses: cachix/install-nix-action@v27 if: | diff --git a/.github/workflows/pages_deploy_main.yaml b/.github/workflows/pages_deploy_main.yaml index 2814d336..caf6b253 100644 --- a/.github/workflows/pages_deploy_main.yaml +++ b/.github/workflows/pages_deploy_main.yaml @@ -30,6 +30,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: cachix/install-nix-action@v27 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pages_requirement_preview.yaml b/.github/workflows/pages_requirement_preview.yaml index 6fddaa4b..df09ab66 100644 --- a/.github/workflows/pages_requirement_preview.yaml +++ b/.github/workflows/pages_requirement_preview.yaml @@ -24,6 +24,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true # -=-=-=-= Strictdoc =-=-=-=- - name: Install python uses: actions/setup-python@v5.1.0 diff --git a/Cargo.toml b/Cargo.toml index 54ed3ec9..c183fc92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,6 @@ hexf = "0.2.1" [features] default = ["hooks"] hooks = [] -spec-test = [] [[bench]] name = "hook_performance_impact" diff --git a/pkgs/wasm-interpreter.nix b/pkgs/wasm-interpreter.nix index 7455665a..2fd7ccff 100644 --- a/pkgs/wasm-interpreter.nix +++ b/pkgs/wasm-interpreter.nix @@ -13,7 +13,7 @@ rustPlatform.buildRustPackage rec { src = ./..; # File suffices to include - extensions = [ "lock" "rs" "toml" ]; + extensions = [ "lock" "rs" "toml" "wast" ]; # Files to explicitly include include = [ ]; # Files to explicitly exclude diff --git a/tests/specification/dummy.wast b/tests/specification/dummy.wast deleted file mode 100644 index 792d949e..00000000 --- a/tests/specification/dummy.wast +++ /dev/null @@ -1,95 +0,0 @@ -;; i32 operations - -(module - (func (export "add") (param $x i32) (param $y i32) (result i32) (i32.add (local.get $x) (local.get $y))) - (func (export "mul") (param $x i32) (param $y i32) (result i32) (i32.mul (local.get $x) (local.get $y))) - (func (export "div_s") (param $x i32) (param $y i32) (result i32) (i32.div_s (local.get $x) (local.get $y))) - (func (export "div_u") (param $x i32) (param $y i32) (result i32) (i32.div_u (local.get $x) (local.get $y))) - (func (export "rem_s") (param $x i32) (param $y i32) (result i32) (i32.rem_s (local.get $x) (local.get $y))) - (func (export "rem_u") (param $x i32) (param $y i32) (result i32) (i32.rem_u (local.get $x) (local.get $y))) -) - -(assert_return (invoke "add" (i32.const 1) (i32.const 1)) (i32.const 2)) -(assert_return (invoke "add" (i32.const 1) (i32.const 0)) (i32.const 1)) -(assert_return (invoke "add" (i32.const -1) (i32.const -1)) (i32.const -2)) -(assert_return (invoke "add" (i32.const -1) (i32.const 1)) (i32.const 0)) -(assert_return (invoke "add" (i32.const 0x7fffffff) (i32.const 1)) (i32.const 0x80000000)) -(assert_return (invoke "add" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x7fffffff)) -(assert_return (invoke "add" (i32.const 0x80000000) (i32.const 0x80000000)) (i32.const 0)) -(assert_return (invoke "add" (i32.const 0x3fffffff) (i32.const 1)) (i32.const 0x40000000)) - -(assert_return (invoke "mul" (i32.const 1) (i32.const 1)) (i32.const 1)) -(assert_return (invoke "mul" (i32.const 1) (i32.const 0)) (i32.const 0)) -(assert_return (invoke "mul" (i32.const -1) (i32.const -1)) (i32.const 1)) -(assert_return (invoke "mul" (i32.const 0x10000000) (i32.const 4096)) (i32.const 0)) -(assert_return (invoke "mul" (i32.const 0x80000000) (i32.const 0)) (i32.const 0)) -(assert_return (invoke "mul" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x80000000)) -(assert_return (invoke "mul" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0x80000001)) -(assert_return (invoke "mul" (i32.const 0x01234567) (i32.const 0x76543210)) (i32.const 0x358e7470)) -(assert_return (invoke "mul" (i32.const 0x7fffffff) (i32.const 0x7fffffff)) (i32.const 1)) - -(assert_return (invoke "div_s" (i32.const 1) (i32.const 1)) (i32.const 1)) -(assert_return (invoke "div_s" (i32.const 0) (i32.const 1)) (i32.const 0)) -(assert_return (invoke "div_s" (i32.const 0) (i32.const -1)) (i32.const 0)) -(assert_return (invoke "div_s" (i32.const -1) (i32.const -1)) (i32.const 1)) -(assert_return (invoke "div_s" (i32.const 0x80000000) (i32.const 2)) (i32.const 0xc0000000)) -(assert_return (invoke "div_s" (i32.const 0x80000001) (i32.const 1000)) (i32.const 0xffdf3b65)) -(assert_return (invoke "div_s" (i32.const 5) (i32.const 2)) (i32.const 2)) -(assert_return (invoke "div_s" (i32.const -5) (i32.const 2)) (i32.const -2)) -(assert_return (invoke "div_s" (i32.const 5) (i32.const -2)) (i32.const -2)) -(assert_return (invoke "div_s" (i32.const -5) (i32.const -2)) (i32.const 2)) -(assert_return (invoke "div_s" (i32.const 7) (i32.const 3)) (i32.const 2)) -(assert_return (invoke "div_s" (i32.const -7) (i32.const 3)) (i32.const -2)) -(assert_return (invoke "div_s" (i32.const 7) (i32.const -3)) (i32.const -2)) -(assert_return (invoke "div_s" (i32.const -7) (i32.const -3)) (i32.const 2)) -(assert_return (invoke "div_s" (i32.const 11) (i32.const 5)) (i32.const 2)) -(assert_return (invoke "div_s" (i32.const 17) (i32.const 7)) (i32.const 2)) - -(assert_return (invoke "div_u" (i32.const 1) (i32.const 1)) (i32.const 1)) -(assert_return (invoke "div_u" (i32.const 0) (i32.const 1)) (i32.const 0)) -(assert_return (invoke "div_u" (i32.const -1) (i32.const -1)) (i32.const 1)) -(assert_return (invoke "div_u" (i32.const 0x80000000) (i32.const -1)) (i32.const 0)) -(assert_return (invoke "div_u" (i32.const 0x80000000) (i32.const 2)) (i32.const 0x40000000)) -(assert_return (invoke "div_u" (i32.const 0x8ff00ff0) (i32.const 0x10001)) (i32.const 0x8fef)) -(assert_return (invoke "div_u" (i32.const 0x80000001) (i32.const 1000)) (i32.const 0x20c49b)) -(assert_return (invoke "div_u" (i32.const 5) (i32.const 2)) (i32.const 2)) -(assert_return (invoke "div_u" (i32.const -5) (i32.const 2)) (i32.const 0x7ffffffd)) -(assert_return (invoke "div_u" (i32.const 5) (i32.const -2)) (i32.const 0)) -(assert_return (invoke "div_u" (i32.const -5) (i32.const -2)) (i32.const 0)) -(assert_return (invoke "div_u" (i32.const 7) (i32.const 3)) (i32.const 2)) -(assert_return (invoke "div_u" (i32.const 11) (i32.const 5)) (i32.const 2)) -(assert_return (invoke "div_u" (i32.const 17) (i32.const 7)) (i32.const 2)) - -(assert_return (invoke "rem_s" (i32.const 0x7fffffff) (i32.const -1)) (i32.const 0)) -(assert_return (invoke "rem_s" (i32.const 1) (i32.const 1)) (i32.const 0)) -(assert_return (invoke "rem_s" (i32.const 0) (i32.const 1)) (i32.const 0)) -(assert_return (invoke "rem_s" (i32.const 0) (i32.const -1)) (i32.const 0)) -(assert_return (invoke "rem_s" (i32.const -1) (i32.const -1)) (i32.const 0)) -(assert_return (invoke "rem_s" (i32.const 0x80000000) (i32.const -1)) (i32.const 0)) -(assert_return (invoke "rem_s" (i32.const 0x80000000) (i32.const 2)) (i32.const 0)) -(assert_return (invoke "rem_s" (i32.const 0x80000001) (i32.const 1000)) (i32.const -647)) -(assert_return (invoke "rem_s" (i32.const 5) (i32.const 2)) (i32.const 1)) -(assert_return (invoke "rem_s" (i32.const -5) (i32.const 2)) (i32.const -1)) -(assert_return (invoke "rem_s" (i32.const 5) (i32.const -2)) (i32.const 1)) -(assert_return (invoke "rem_s" (i32.const -5) (i32.const -2)) (i32.const -1)) -(assert_return (invoke "rem_s" (i32.const 7) (i32.const 3)) (i32.const 1)) -(assert_return (invoke "rem_s" (i32.const -7) (i32.const 3)) (i32.const -1)) -(assert_return (invoke "rem_s" (i32.const 7) (i32.const -3)) (i32.const 1)) -(assert_return (invoke "rem_s" (i32.const -7) (i32.const -3)) (i32.const -1)) -(assert_return (invoke "rem_s" (i32.const 11) (i32.const 5)) (i32.const 1)) -(assert_return (invoke "rem_s" (i32.const 17) (i32.const 7)) (i32.const 3)) - -(assert_return (invoke "rem_u" (i32.const 1) (i32.const 1)) (i32.const 0)) -(assert_return (invoke "rem_u" (i32.const 0) (i32.const 1)) (i32.const 0)) -(assert_return (invoke "rem_u" (i32.const -1) (i32.const -1)) (i32.const 0)) -(assert_return (invoke "rem_u" (i32.const 0x80000000) (i32.const -1)) (i32.const 0x80000000)) -(assert_return (invoke "rem_u" (i32.const 0x80000000) (i32.const 2)) (i32.const 0)) -(assert_return (invoke "rem_u" (i32.const 0x8ff00ff0) (i32.const 0x10001)) (i32.const 0x8001)) -(assert_return (invoke "rem_u" (i32.const 0x80000001) (i32.const 1000)) (i32.const 649)) -(assert_return (invoke "rem_u" (i32.const 5) (i32.const 2)) (i32.const 1)) -(assert_return (invoke "rem_u" (i32.const -5) (i32.const 2)) (i32.const 1)) -(assert_return (invoke "rem_u" (i32.const 5) (i32.const -2)) (i32.const 5)) -(assert_return (invoke "rem_u" (i32.const -5) (i32.const -2)) (i32.const -5)) -(assert_return (invoke "rem_u" (i32.const 7) (i32.const 3)) (i32.const 1)) -(assert_return (invoke "rem_u" (i32.const 11) (i32.const 5)) (i32.const 1)) -(assert_return (invoke "rem_u" (i32.const 17) (i32.const 7)) (i32.const 3)) diff --git a/tests/specification/mod.rs b/tests/specification/mod.rs index 65cc75e7..1874d3c5 100644 --- a/tests/specification/mod.rs +++ b/tests/specification/mod.rs @@ -7,21 +7,38 @@ mod reports; mod run; mod test_errors; -#[test_log::test] -pub fn spec_dummy() { - let path = "./tests/specification/dummy.wast"; - let report = run::run_spec_test(path); - println!("Report for {}:\n{}", path, report); -} - #[test_log::test] pub fn spec_tests() { let paths = get_wast_files(Path::new("./tests/specification/testsuite/")) .expect("Failed to find testsuite"); + + let mut successful_reports = 0; + let mut failed_reports = 0; + let mut compile_error_reports = 0; + for test_path in paths { + println!("Report for {}:", test_path.display()); let report = run::run_spec_test(test_path.to_str().unwrap()); - println!("Report for {}:\n{}", test_path.display(), report); + println!("{}", report); + + match report { + reports::WastTestReport::Asserts(assert_report) => { + if assert_report.has_errors() { + failed_reports += 1; + } else { + successful_reports += 1; + } + } + reports::WastTestReport::CompilationError(_) => { + compile_error_reports += 1; + } + } } + + println!( + "Tests: {} Passed, {} Failed, {} Compilation Errors", + successful_reports, failed_reports, compile_error_reports + ); } // See: https://stackoverflow.com/a/76820878 diff --git a/tests/specification/reports.rs b/tests/specification/reports.rs index af6cfb32..e4f0f54b 100644 --- a/tests/specification/reports.rs +++ b/tests/specification/reports.rs @@ -1,135 +1,143 @@ use std::error::Error; pub struct WastSuccess { - filename: String, line_number: u32, command: String, } +impl WastSuccess { + pub fn new(line_number: u32, command: &str) -> Self { + Self { + line_number, + command: command.to_string(), + } + } +} + pub struct WastError { inner: Box, - filename: String, - line_number: u32, + line_number: Option, command: String, } impl WastError { - pub fn new(error: Box, filename: String, line_number: u32, command: &str) -> Self { + pub fn new(error: Box, line_number: u32, command: &str) -> Self { Self { inner: error, - filename, - line_number, + line_number: Some(line_number), command: command.to_string(), } } - - pub fn from_outside(error: Box, reason: &str) -> Self { - Self { - inner: error, - filename: "".to_string(), - line_number: 0, - command: reason.to_string(), - } - } } pub struct AssertReport { + filename: String, results: Vec>, } impl AssertReport { - pub fn new() -> Self { + pub fn new(filename: &str) -> Self { Self { + filename: filename.to_string(), results: Vec::new(), } } - pub fn push_success(&mut self, filename: String, line_number: u32, command: String) { - self.results.push(Ok(WastSuccess { - filename, - line_number, - command, - })); + pub fn push_success(&mut self, success: WastSuccess) { + self.results.push(Ok(success)); } - pub fn push_error( - &mut self, - filename: String, - line_number: u32, - command: String, - error: Box, - ) { - self.results.push(Err(WastError { - inner: error, - filename, - line_number, - command, - })); + pub fn push_error(&mut self, error: WastError) { + self.results.push(Err(error)); } -} -pub enum WastTestReport { - Asserts(AssertReport), - CompilationError(WastError), -} - -impl From for WastTestReport { - fn from(error: WastError) -> Self { - WastTestReport::CompilationError(error) + pub fn compile_report(self) -> WastTestReport { + return WastTestReport::Asserts(self); } -} -impl From for WastTestReport { - fn from(assert_report: AssertReport) -> Self { - WastTestReport::Asserts(assert_report) + pub fn has_errors(&self) -> bool { + self.results.iter().any(|r| r.is_err()) } } -// .------------------------. -// | Display Implementation | -// '------------------------' - -impl std::fmt::Display for WastSuccess { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "[SUCCESS] {} ({}:{})", - self.command, self.filename, self.line_number - ) - } +pub struct CompilationError { + inner: Box, + filename: String, + context: String, } -impl std::fmt::Display for WastError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!( - f, - "[ERROR] {} ({}:{})", - self.command, self.filename, self.line_number - )?; - write!(f, "\t{}", self.inner)?; +impl CompilationError { + pub fn new(error: Box, filename: &str, context: &str) -> Self { + Self { + inner: error, + filename: filename.to_string(), + context: context.to_string(), + } + } - Ok(()) + pub fn compile_report(self) -> WastTestReport { + return WastTestReport::CompilationError(self); } } -impl std::fmt::Display for AssertReport { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for result in &self.results { - match result { - Ok(success) => writeln!(f, "{}", success)?, - Err(error) => writeln!(f, "{}", error)?, - } - } - - Ok(()) - } +pub enum WastTestReport { + Asserts(AssertReport), + CompilationError(CompilationError), } +// .------------------------. +// | Display Implementation | +// '------------------------' + impl std::fmt::Display for WastTestReport { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - WastTestReport::Asserts(assert_report) => write!(f, "{}", assert_report), - WastTestReport::CompilationError(error) => write!(f, "{}", error), + WastTestReport::CompilationError(error) => { + writeln!(f, "------ {} ------", error.filename)?; + writeln!(f, "⚠ Compilation Failed ⚠")?; + writeln!(f, "Context: {}", error.context)?; + writeln!(f, "Error: {}", error.inner)?; + writeln!(f, "~~~~~~~~~~~~~~~~")?; + writeln!(f, "")?; + } + WastTestReport::Asserts(assert_report) => { + writeln!(f, "------ {} ------", assert_report.filename)?; + for result in &assert_report.results { + match result { + Ok(success) => { + writeln!( + f, + "✅ {}:{} -> {}", + assert_report.filename, success.line_number, success.command + )?; + } + Err(error) => { + writeln!( + f, + "❌ {}:{} -> {}", + assert_report.filename, + error.line_number.unwrap_or(0), + error.command + )?; + writeln!(f, " Error: {}", error.inner)?; + } + } + } + let passed_asserts = assert_report.results.iter().filter(|r| r.is_ok()).count(); + let failed_asserts = assert_report.results.iter().filter(|r| r.is_err()).count(); + let total_asserts = assert_report.results.len(); + + writeln!(f, "")?; + writeln!( + f, + "Execution finished. Passed: {}, Failed: {}, Total: {}", + passed_asserts, failed_asserts, total_asserts + )?; + writeln!(f, "~~~~~~~~~~~~~~~~")?; + writeln!(f, "")?; + } } + + Ok(()) } } diff --git a/tests/specification/run.rs b/tests/specification/run.rs index a856efd0..7c90c124 100644 --- a/tests/specification/run.rs +++ b/tests/specification/run.rs @@ -4,32 +4,60 @@ use std::panic::AssertUnwindSafe; use wasm::Value; use wasm::{validate, RuntimeInstance}; +use wast::core::WastArgCore; +use wast::core::WastRetCore; +use wast::WastArg; use crate::specification::reports::*; use crate::specification::test_errors::*; +/// Attempt to unwrap the result of an expression. If the expression is an `Err`, then `return` the +/// error. +/// +/// # Motivation +/// The `Try` trait is not yet stable, so we define our own macro to simulate the `Result` type. macro_rules! try_to { ($e:expr) => { match $e { Ok(val) => val, - Err(err) => return err.into(), + Err(err) => return err, } }; } pub fn run_spec_test(filepath: &str) -> WastTestReport { // -=-= Initialization =-=- - let contents = try_to!(std::fs::read_to_string(filepath) - .map_err(|err| WastError::from_outside(Box::new(err), "failed to open wast file"))); + let contents = try_to!( + std::fs::read_to_string(filepath).map_err(|err| CompilationError::new( + Box::new(err), + filepath, + "failed to open wast file" + ) + .compile_report()) + ); - let buf = try_to!(wast::parser::ParseBuffer::new(&contents) - .map_err(|err| WastError::from_outside(Box::new(err), "failed to create wast buffer"))); + let buf = + try_to!( + wast::parser::ParseBuffer::new(&contents).map_err(|err| CompilationError::new( + Box::new(err), + filepath, + "failed to create wast buffer" + ) + .compile_report()) + ); - let wast = try_to!(wast::parser::parse::(&buf) - .map_err(|err| WastError::from_outside(Box::new(err), "failed to parse wast file"))); + let wast = + try_to!( + wast::parser::parse::(&buf).map_err(|err| CompilationError::new( + Box::new(err), + filepath, + "failed to parse wast file" + ) + .compile_report()) + ); // -=-= Testing & Compilation =-=- - let mut asserts = AssertReport::new(); + let mut asserts = AssertReport::new(filepath); // We need to keep the wasm_bytes in-scope for the lifetime of the interpeter. // As such, we hoist the bytes into an Option, and assign it once a module directive is found. @@ -40,47 +68,54 @@ pub fn run_spec_test(filepath: &str) -> WastTestReport { for directive in wast.directives { match directive { wast::WastDirective::Wat(mut quoted) => { - let encoded = try_to!(quoted - .encode() - .map_err(|err| WastError::from_outside(Box::new(err), "failed to encode wat"))); + // If we fail to compile or to validate the main module, then we should treat this + // as a fatal (compilation) error. + let encoded = try_to!(quoted.encode().map_err(|err| CompilationError::new( + Box::new(err), + filepath, + "failed to encode main module's wat" + ) + .compile_report())); wasm_bytes = Some(encoded); let validation_attempt = catch_unwind(|| { validate(wasm_bytes.as_ref().unwrap()).map_err(|err| { - WastError::new( - Box::::new(err.into()), - filepath.to_string(), - 0, - "Module validation failed", + CompilationError::new( + Box::new(WasmInterpreterError(err)), + filepath, + "main module validation failed", ) + .compile_report() }) }); let validation_info = match validation_attempt { Ok(original_result) => try_to!(original_result), - Err(inner) => { + Err(panic) => { // TODO: Do we want to exit on panic? State may be left in an inconsistent state, and cascading panics may occur. - let err = if let Ok(msg) = inner.downcast::<&str>() { - Box::new(PanicError::new(msg.to_string())) + let err = if let Ok(msg) = panic.downcast::<&str>() { + Box::new(PanicError::new(&msg)) } else { - Box::new(PanicError::new("Unknown panic".into())) + Box::new(PanicError::new("Unknown panic")) }; - return WastError::new( + return CompilationError::new( err, - filepath.to_string(), - 0, - "Module validation panicked", + filepath, + "main module validation panicked", ) - .into(); + .compile_report(); } }; let instance = try_to!(RuntimeInstance::new(&validation_info).map_err(|err| { - let err: wasm::Error = err.into(); - let err: WasmInterpreterError = err.into(); - WastError::from_outside(Box::new(err), "failed to create runtime instance") + CompilationError::new( + Box::new(WasmInterpreterError(wasm::Error::RuntimeError(err))), + filepath, + "failed to create runtime instance", + ) + .compile_report() })); interpeter = Some(instance); @@ -91,15 +126,14 @@ pub fn run_spec_test(filepath: &str) -> WastTestReport { results, } => { if interpeter.is_none() { - asserts.push_error( - filepath.to_string(), - span.linecol_in(&contents).0 as u32 + 1, - get_command(&contents, span), + return CompilationError::new( Box::new(GenericError::new( - "Attempted to run assert_return before a module was compiled", + "Attempted to assert before module directive", )), - ); - continue; + filepath, + "no module directive found", + ) + .compile_report(); } let interpeter = interpeter.as_mut().unwrap(); @@ -112,28 +146,26 @@ pub fn run_spec_test(filepath: &str) -> WastTestReport { Err(inner) => { // TODO: Do we want to exit on panic? State may be left in an inconsistent state, and cascading panics may occur. if let Ok(msg) = inner.downcast::<&str>() { - Err(Box::new(PanicError::new(msg.to_string()))) + Err(Box::new(PanicError::new(&msg))) } else { - Err(Box::new(PanicError::new("Unknown panic".into()))) + Err(Box::new(PanicError::new("Unknown panic"))) } } }; match err_or_panic { Ok(_) => { - asserts.push_success( - filepath.to_string(), + asserts.push_success(WastSuccess::new( span.linecol_in(&contents).0 as u32 + 1, get_command(&contents, span), - ); + )); } Err(inner) => { - asserts.push_error( - filepath.to_string(), + asserts.push_error(WastError::new( + inner, span.linecol_in(&contents).0 as u32 + 1, get_command(&contents, span), - inner, - ); + )); } } } @@ -168,20 +200,37 @@ pub fn run_spec_test(filepath: &str) -> WastTestReport { message: _, } | wast::WastDirective::AssertException { span, exec: _ } => { - asserts.push_error( - filepath.to_string(), + asserts.push_error(WastError::new( + Box::new(GenericError::new("Assert directive not yet implemented")), span.linecol_in(&contents).0 as u32 + 1, get_command(&contents, span), - Box::new(GenericError::new("Assert directive not yet implemented")), - ); + )); + } + wast::WastDirective::Wait { span, thread: _ } => { + asserts.push_error(WastError::new( + Box::new(GenericError::new("Wait directive not yet implemented")), + span.linecol_in(&contents).0 as u32 + 1, + get_command(&contents, span), + )); + } + wast::WastDirective::Invoke(invoke) => { + asserts.push_error(WastError::new( + Box::new(GenericError::new("Invoke directive not yet implemented")), + invoke.span.linecol_in(&contents).0 as u32 + 1, + get_command(&contents, invoke.span), + )); + } + wast::WastDirective::Thread(thread) => { + asserts.push_error(WastError::new( + Box::new(GenericError::new("Thread directive not yet implemented")), + thread.span.linecol_in(&contents).0 as u32 + 1, + get_command(&contents, thread.span), + )); } - wast::WastDirective::Wait { span: _, thread: _ } => todo!(), - wast::WastDirective::Invoke(_) => todo!(), - wast::WastDirective::Thread(_) => todo!(), } } - asserts.into() + asserts.compile_report() } fn execute_assert_return( @@ -205,11 +254,7 @@ fn execute_assert_return( let actual = interpeter .invoke_named_dynamic(invoke_info.name, args, &result_types) - .map_err(|err| { - let err: wasm::Error = err.into(); - let err: WasmInterpreterError = err.into(); - Box::new(err) - })?; + .map_err(|err| Box::new(WasmInterpreterError(wasm::Error::RuntimeError(err))))?; AssertEqError::assert_eq(actual, result_vals)?; } @@ -217,64 +262,81 @@ fn execute_assert_return( span: _, module: _, global: _, - } => todo!(), - wast::WastExecute::Wat(_) => todo!(), + } => todo!("`get` directive inside `assert_return` not yet implemented"), + wast::WastExecute::Wat(_) => { + todo!("`wat` directive inside `assert_return` not yet implemented") + } } Ok(()) } -pub fn arg_to_value(arg: wast::WastArg) -> Value { +pub fn arg_to_value(arg: WastArg) -> Value { match arg { - wast::WastArg::Core(core_arg) => match core_arg { - wast::core::WastArgCore::I32(val) => Value::I32(val as u32), - wast::core::WastArgCore::I64(val) => Value::I64(val as u64), - wast::core::WastArgCore::F32(val) => Value::F32(wasm::value::F32(val.bits as f32)), - wast::core::WastArgCore::F64(val) => Value::F64(wasm::value::F64(val.bits as f64)), - wast::core::WastArgCore::V128(_) => todo!(), - wast::core::WastArgCore::RefNull(_) => todo!(), - wast::core::WastArgCore::RefExtern(_) => todo!(), - wast::core::WastArgCore::RefHost(_) => todo!(), + WastArg::Core(core_arg) => match core_arg { + WastArgCore::I32(val) => Value::I32(val as u32), + WastArgCore::I64(val) => Value::I64(val as u64), + WastArgCore::F32(val) => Value::F32(wasm::value::F32(val.bits as f32)), + WastArgCore::F64(val) => Value::F64(wasm::value::F64(val.bits as f64)), + WastArgCore::V128(_) => todo!("`V128` value arguments not yet implemented"), + WastArgCore::RefNull(_) => { + todo!("`RefNull` value arguments not yet implemented") + } + WastArgCore::RefExtern(_) => { + todo!("`RefExtern` value arguments not yet implemented") + } + WastArgCore::RefHost(_) => { + todo!("`RefHost` value arguments not yet implemented") + } }, - wast::WastArg::Component(_) => todo!(), + WastArg::Component(_) => todo!("`Component` value arguments not yet implemented"), } } fn result_to_value(result: wast::WastRet) -> Value { match result { wast::WastRet::Core(core_arg) => match core_arg { - wast::core::WastRetCore::I32(val) => Value::I32(val as u32), - wast::core::WastRetCore::I64(val) => Value::I64(val as u64), - wast::core::WastRetCore::F32(val) => match val { - wast::core::NanPattern::CanonicalNan => todo!(), - wast::core::NanPattern::ArithmeticNan => todo!(), + WastRetCore::I32(val) => Value::I32(val as u32), + WastRetCore::I64(val) => Value::I64(val as u64), + WastRetCore::F32(val) => match val { + wast::core::NanPattern::CanonicalNan => { + todo!("`F32::CanonicalNan` result not yet implemented") + } + wast::core::NanPattern::ArithmeticNan => { + todo!("`F32::ArithmeticNan` result not yet implemented") + } wast::core::NanPattern::Value(val) => Value::F32(wasm::value::F32(val.bits as f32)), }, - wast::core::WastRetCore::F64(val) => match val { - wast::core::NanPattern::CanonicalNan => todo!(), - wast::core::NanPattern::ArithmeticNan => todo!(), + WastRetCore::F64(val) => match val { + wast::core::NanPattern::CanonicalNan => { + todo!("`F64::CanonicalNan` result not yet implemented") + } + wast::core::NanPattern::ArithmeticNan => { + todo!("`F64::ArithmeticNan` result not yet implemented") + } wast::core::NanPattern::Value(val) => Value::F64(wasm::value::F64(val.bits as f64)), }, - wast::core::WastRetCore::V128(_) => todo!(), - wast::core::WastRetCore::RefNull(_) => todo!(), - wast::core::WastRetCore::RefExtern(_) => todo!(), - wast::core::WastRetCore::RefHost(_) => todo!(), - wast::core::WastRetCore::RefFunc(_) => todo!(), - wast::core::WastRetCore::RefAny => todo!(), - wast::core::WastRetCore::RefEq => todo!(), - wast::core::WastRetCore::RefArray => todo!(), - wast::core::WastRetCore::RefStruct => todo!(), - wast::core::WastRetCore::RefI31 => todo!(), - wast::core::WastRetCore::Either(_) => todo!(), + WastRetCore::V128(_) => todo!("`V128` result not yet implemented"), + WastRetCore::RefNull(_) => todo!("`RefNull` result not yet implemented"), + WastRetCore::RefExtern(_) => { + todo!("`RefExtern` result not yet implemented") + } + WastRetCore::RefHost(_) => todo!("`RefHost` result not yet implemented"), + WastRetCore::RefFunc(_) => todo!("`RefFunc` result not yet implemented"), + WastRetCore::RefAny => todo!("`RefAny` result not yet implemented"), + WastRetCore::RefEq => todo!("`RefEq` result not yet implemented"), + WastRetCore::RefArray => todo!("`RefArray` result not yet implemented"), + WastRetCore::RefStruct => todo!("`RefStruct` result not yet implemented"), + WastRetCore::RefI31 => todo!("`RefI31` result not yet implemented"), + WastRetCore::Either(_) => todo!("`Either` result not yet implemented"), }, - wast::WastRet::Component(_) => todo!(), + wast::WastRet::Component(_) => todo!("`Component` result not yet implemented"), } } -pub fn get_command(contents: &str, span: wast::token::Span) -> String { +pub fn get_command(contents: &str, span: wast::token::Span) -> &str { contents[span.offset() as usize..] .lines() .next() .unwrap_or("") - .to_string() } diff --git a/tests/specification/test_errors.rs b/tests/specification/test_errors.rs index 0c94cfc7..55101557 100644 --- a/tests/specification/test_errors.rs +++ b/tests/specification/test_errors.rs @@ -35,8 +35,10 @@ pub struct PanicError { } impl PanicError { - pub fn new(message: String) -> Self { - PanicError { message } + pub fn new(message: &str) -> Self { + PanicError { + message: message.to_string(), + } } } @@ -48,7 +50,7 @@ impl std::fmt::Display for PanicError { } #[derive(Debug, PartialEq, Eq, Clone)] -pub struct WasmInterpreterError(wasm::Error); +pub struct WasmInterpreterError(pub wasm::Error); impl Error for WasmInterpreterError { fn source(&self) -> Option<&(dyn Error + 'static)> { @@ -59,15 +61,9 @@ impl Error for WasmInterpreterError { } } -impl From for WasmInterpreterError { - fn from(error: wasm::Error) -> Self { - WasmInterpreterError(error) - } -} - impl std::fmt::Display for WasmInterpreterError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "{}", self.0) + write!(f, "{}", self.0) } } diff --git a/tests/specification/testsuite b/tests/specification/testsuite index 53da17c0..7c3ec23a 160000 --- a/tests/specification/testsuite +++ b/tests/specification/testsuite @@ -1 +1 @@ -Subproject commit 53da17c0936a23f68f97cde4f9346a0a374dc35f +Subproject commit 7c3ec23ab19b37c68976b555f9491752cbda6d5f diff --git a/tests/wasm_spec_testsuite.rs b/tests/wasm_spec_testsuite.rs index 0fdfb322..377d7875 100644 --- a/tests/wasm_spec_testsuite.rs +++ b/tests/wasm_spec_testsuite.rs @@ -1,5 +1,4 @@ // The reason this file exists is only to expose the `specification` module to the outside world. // More so, the reason it wasn't added to the `lib.rs` file is because we wanted to separate the // regular tests from the spec tests. -#[cfg(feature = "spec-test")] mod specification;