stash

tests/specification/mod.rs

@@ -1,6 +1,157 @@
+use std::{error::Error, fs, path::{Path, PathBuf}};
+
 mod run;
 
 #[test_log::test]
-pub fn spec_i32() {
-    run::run_spec_test("./tests/specification/testsuite/../dummy.wast");
+pub fn spec_dummy() {
+    let path = "./tests/specification/dummy.wast";
+    let report = run::run_spec_test(path);
+    println!("Report for {}:\n{}", path, report);
+}
+
+#[test_log::test]
+pub fn spec_tests() {
+    let paths = get_wast_files(Path::new("./tests/specification/testsuite/")).expect("Failed to find testsuite");
+    for test_path in paths {
+        let report = run::run_spec_test(test_path.to_str().unwrap());
+        println!("Report for {}:\n{}", test_path.display(), report);
+    }
+}
+
+// See: https://stackoverflow.com/a/76820878
+fn get_wast_files(base_path: &Path) -> Result<Vec<PathBuf>, std::io::Error> {
+    let mut buf = vec![];
+    let entries = fs::read_dir(base_path)?;
+
+    for entry in entries {
+        let entry = entry?;
+        let meta = entry.metadata()?;
+
+        if meta.is_dir() {
+            let mut subdir = get_wast_files(&entry.path())?;
+            buf.append(&mut subdir);
+        }
+
+        if meta.is_file() && entry.path().extension().unwrap_or_default() == "wast" {
+            buf.push(entry.path());
+        }
+    }
+
+    Ok(buf)
+}
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+struct WasmErrorWrapper(wasm::Error);
+
+impl Error for WasmErrorWrapper {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        match &self.0 {
+            wasm::Error::MalformedUtf8String(inner) => Some(inner),
+            _ => None
+        }
+    }
+}
+
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+struct AssertEqError {
+    left: String,
+    right: String,
+}
+
+impl AssertEqError {
+    pub fn assert_eq<T: std::fmt::Debug + PartialEq>(left: T, right: T) -> Result<(), Self> {
+        if left != right {
+            return Err(AssertEqError {
+                left: format!("{:?}", left),
+                right: format!("{:?}", right),
+            });
+        }
+
+        Ok(())
+    }
+}
+impl Error for AssertEqError {}
+impl std::fmt::Display for AssertEqError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "assert_eq failed: left: {}, right: {}", self.left, self.right)
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+struct PanicError {
+    message: String,
 }
+
+impl PanicError {
+    pub fn new(message: String) -> Self {
+        PanicError { message }
+    }
+}
+
+impl Error for PanicError {}
+impl std::fmt::Display for PanicError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Panic: {}", self.message)
+    }
+}
+
+impl std::fmt::Display for WasmErrorWrapper {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "{}", self.0)
+    }
+
+}
+
+type AssertReport = Vec<Result<WastSuccess, WastError>>;
+
+type WastTestReport = Result<AssertReport, WastError>;
+
+struct WastSuccess {
+    filename: String,
+    line_number: u32,
+    command: String,
+}
+
+struct WastError {
+    inner: Box<dyn Error>,
+    filename: String,
+    line_number: u32,
+    command: String,
+}
+
+pub fn get_command(contents: &str, span: wast::token::Span) -> String {
+    contents[span.offset() as usize..].lines().next().unwrap_or("<unknown>").to_string()
+}
+
+impl std::fmt::Display for WastError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "[ERROR] {} ({}:{})", self.command, self.filename, self.line_number)?;
+        writeln!(f, "\t{}", self.inner)?;
+
+        Ok(())
+    }
+}
+
+impl std::fmt::Display for WastSuccess {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "[SUCCESS] {} ({}:{})", self.command, self.filename, self.line_number)
+    }
+}
+
+pub fn display_report(report: WastTestReport) -> std::fmt::Result {
+    let mut f:std::fmt::Formatter<'_> = std::fmt::Formatter::new();
+    match self {
+        WastTestReport::Asserts(asserts) => {
+            for assert in asserts {
+                match assert {
+                    Ok(success) => write!(f, "{}", success),
+                    Err(error) => write!(f, "{}", error),
+                }?;
+            }
+        },
+        WastTestReport::CompilationError(error) => write!(f, "{}", error)?,
+    }
+
+    Ok(())
+}

tests/specification/run.rs

@@ -1,15 +1,48 @@
-use log::debug;
+use std::panic::{catch_unwind, AssertUnwindSafe};
+
 use wasm::{
     validate,
     value::{InteropValueList, Value},
     RuntimeInstance,
 };
 
+use crate::specification::AssertEqError;
+
+use super::{get_command, AssertReport, PanicError, WasmErrorWrapper, WastError, WastSuccess, WastTestReport};
+
+
+pub fn run_spec_test(filepath: &str) -> WastTestReport{
+    let contents = match std::fs::read_to_string(filepath) {
+        Ok(contents) => contents,
+        Err(inner) => return WastTestReport::CompilationError(WastError {
+            inner: Box::new(inner),
+            filename: filepath.to_string(),
+            line_number: 0,
+            command: "<read from file>".to_string(),
+        }),
+    };
+
+    let buf = match wast::parser::ParseBuffer::new(&contents)  {
+        Ok(contents) => contents,
+        Err(inner) => return WastTestReport::CompilationError(WastError {
+            inner: Box::new(inner),
+            filename: filepath.to_string(),
+            line_number: 0,
+            command: "<wast lexer failed>".to_string(),
+        }),
+    };
+
+    let wast = match wast::parser::parse::<wast::Wast>(&buf) {
+        Ok(contents) => contents,
+        Err(inner) => return WastTestReport::CompilationError(WastError {
+            inner: Box::new(inner),
+            filename: filepath.to_string(),
+            line_number: 0,
+            command: "<wast parser failed>".to_string(),
+        }),
+    };
 
-pub fn run_spec_test(filepath: &str) {
-    let contents = std::fs::read_to_string(filepath).expect("Failed to read .wast file");
-    let buf = wast::parser::ParseBuffer::new(&contents).expect("Lexer failed for .wast file");
-    let wast = wast::parser::parse::<wast::Wast>(&buf).expect("Failed to parse .wast file");
+    let mut assert_report: AssertReport = vec![];
 
     // We need to keep the wasm_bytes in-scope for the lifetime of the interpeter.
     // As such, we hoist the bytes into an Option, and assign it once a module directive is found.
@@ -20,45 +53,90 @@ pub fn run_spec_test(filepath: &str) {
     for directive in wast.directives {
         match directive {
             wast::WastDirective::Wat(mut quoted) => {
-                wasm_bytes = Some(quoted.encode().expect("Failed to encode module into bytes"));
+                wasm_bytes = Some(match quoted.encode().map_err(|inner| WastTestReport::CompilationError(WastError {
+                    inner: Box::new(inner),
+                    filename: filepath.to_string(),
+                    line_number: 0,
+                    command: "<wat encoder failed>".to_string(),
+                })));
+
                 let validation_info =
-                    validate(wasm_bytes.as_ref().unwrap()).expect("validation failed");
+                    match validate(wasm_bytes.as_ref().unwrap()) {
+                        Ok(info) => info,
+                        Err(inner) => return WastTestReport::CompilationError(WastError {
+                            inner: Box::new(WasmErrorWrapper(inner)),
+                            filename: filepath.to_string(),
+                            line_number: 0,
+                            command: "<wasm validation failed>".to_string(),
+                        }),
+                    };
+
                 interpeter =
-                    Some(RuntimeInstance::new(&validation_info).expect("instantiation failed"));
+                    Some(match RuntimeInstance::new(&validation_info) {
+                        Ok(interpeter) => interpeter,
+                        Err(inner) => return WastTestReport::CompilationError(WastError {
+                            inner: Box::new(WasmErrorWrapper(inner)),
+                            filename: filepath.to_string(),
+                            line_number: 0,
+                            command: "<wasm interpeter creation failed>".to_string(),
+                        }),
+                    });
             }
             wast::WastDirective::AssertReturn {
-                span: _,
+                span,
                 exec,
                 results,
             } => {
                 if let Some(interpeter) = interpeter.as_mut() {
-                    execute_assert_return(interpeter, exec, Results::new(results));
+                    let err_or_panic: Result<(), Box<dyn std::error::Error>> = match catch_unwind(AssertUnwindSafe(|| execute_assert_return(interpeter, exec, Results::new(results)))) {
+                        Ok(original_result) => original_result,
+                        Err(inner) => {
+                            // TODO: Do we want to exit on panic? State may be left in an inconsistent state, and cascading panics may occur.
+                            if let Ok(msg) = inner.downcast::<&str>() {
+                                Err(Box::new(PanicError::new(msg.to_string())))
+                            } else {
+                                Err(Box::new(PanicError::new("Unknown panic".into())))
+                            }
+                        },
+                    };
+                    let rez = err_or_panic.map_err(|inner| {
+                        WastError {
+                            inner: inner,
+                            filename: filepath.to_string(),
+                            line_number: span.linecol_in(&contents).0 as u32 + 1,
+                            command: get_command(&contents, span)
+                        }
+                    }).map(|_| WastSuccess {
+                        filename: filepath.to_string(),
+                        line_number: span.linecol_in(&contents).0 as u32 + 1,
+                        command: get_command(&contents, span)
+                    });
+
+                    assert_report.push(rez);
                 } else {
                     todo!()
                 }
             }
             _ => todo!("{:?}", directive),
         }
-    }
+    };
+
+    WastTestReport::Asserts(assert_report)
 }
 
 pub fn execute_assert_return<Res: InteropValueList>(
     interpeter: &mut RuntimeInstance,
     exec: wast::WastExecute,
     results: Res,
-) {
+) -> Result<(), Box<dyn std::error::Error>> {
     match exec {
         wast::WastExecute::Invoke(invoke_info) => {
             let actual = interpeter
-                .invoke_named::<_, Res>(invoke_info.name, Args::new(invoke_info.args))
-                .unwrap().into_values();
+                .invoke_named::<_, Res>(invoke_info.name, Args::new(invoke_info.args)).map_err(WasmErrorWrapper)?.into_values();
 
             let expected = results.into_values();
 
-            debug!("actual: {:?}", actual);
-            debug!("expected: {:?}", expected);
-
-            assert_eq!(actual, expected);
+            AssertEqError::assert_eq(actual, expected)?;
         }
         wast::WastExecute::Get {
             span: _,
@@ -67,6 +145,8 @@ pub fn execute_assert_return<Res: InteropValueList>(
         } => todo!(),
         wast::WastExecute::Wat(_) => todo!(),
     }
+
+    Ok(())
 }
 
 struct Args<'a> {