refactor: refactor spec testsuite and remove feature

Signed-off-by: George Cosma <[email protected]>

.github/workflows/ci.yml

@@ -21,6 +21,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - name: Format
         run: cargo check
       - name: Run clippy
@@ -39,11 +41,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: webiny/[email protected]
 
   msrv:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: taiki-e/install-action@cargo-hack
       - run: cargo hack check --rust-version --workspace --all-targets --ignore-private

.github/workflows/nix.yml

@@ -11,6 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: cachix/install-nix-action@v27
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
@@ -26,22 +28,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: cachix/install-nix-action@v27
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
       - uses: cachix/cachix-action@v14
         with:
           name: dlr-ft
           authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
-      - run: nix build .#wasm-interpreter --print-build-logs
+      - run: nix build .?submodules=1#wasm-interpreter --print-build-logs
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
           verbose: true
           file: result/lcov-codecov.json
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      - run: nix build .#report --print-build-logs
+      - run: nix build .?submodules=1#report --print-build-logs
       - name: Archive report
         uses: actions/upload-artifact@v4
         with:

.github/workflows/pages_coverage_preview.yaml

@@ -23,6 +23,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       # -=-=-=-= Create report =-=-=-=-
       - uses: cachix/install-nix-action@v27
         if: |

.github/workflows/pages_deploy_main.yaml

@@ -30,6 +30,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: cachix/install-nix-action@v27
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pages_requirement_preview.yaml

@@ -24,6 +24,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       # -=-=-=-= Strictdoc =-=-=-=-
       - name: Install python
         uses: actions/[email protected]

Cargo.toml

@@ -35,7 +35,6 @@ hexf = "0.2.1"
 [features]
 default = ["hooks"]
 hooks = []
-spec-test = []
 
 [[bench]]
 name = "hook_performance_impact"

pkgs/wasm-interpreter.nix

@@ -13,7 +13,7 @@ rustPlatform.buildRustPackage rec {
       src = ./..;
 
       # File suffices to include
-      extensions = [ "lock" "rs" "toml" ];
+      extensions = [ "lock" "rs" "toml" "wast" ];
       # Files to explicitly include
       include = [ ];
       # Files to explicitly exclude

tests/specification/mod.rs

@@ -18,10 +18,34 @@ pub fn spec_dummy() {
 pub fn spec_tests() {
     let paths = get_wast_files(Path::new("./tests/specification/testsuite/"))
         .expect("Failed to find testsuite");
+
+    let mut successful_reports = 0;
+    let mut failed_reports = 0;
+    let mut compile_error_reports = 0;
+
     for test_path in paths {
+        println!("Report for {}:", test_path.display());
         let report = run::run_spec_test(test_path.to_str().unwrap());
-        println!("Report for {}:\n{}", test_path.display(), report);
+        println!("{}", report);
+
+        match report {
+            reports::WastTestReport::Asserts(assert_report) => {
+                if assert_report.has_errors() {
+                    failed_reports += 1;
+                } else {
+                    successful_reports += 1;
+                }
+            }
+            reports::WastTestReport::CompilationError(_) => {
+                compile_error_reports += 1;
+            }
+        }
     }
+
+    println!(
+        "Tests: {} Passed, {} Failed, {} Compilation Errors",
+        successful_reports, failed_reports, compile_error_reports
+    );
 }
 
 // See: https://stackoverflow.com/a/76820878

tests/specification/reports.rs

@@ -1,135 +1,143 @@
 use std::error::Error;
 
 pub struct WastSuccess {
-    filename: String,
     line_number: u32,
     command: String,
 }
 
+impl WastSuccess {
+    pub fn new(line_number: u32, command: &str) -> Self {
+        Self {
+            line_number,
+            command: command.to_string(),
+        }
+    }
+}
+
 pub struct WastError {
     inner: Box<dyn Error>,
-    filename: String,
-    line_number: u32,
+    line_number: Option<u32>,
     command: String,
 }
 
 impl WastError {
-    pub fn new(error: Box<dyn Error>, filename: String, line_number: u32, command: &str) -> Self {
+    pub fn new(error: Box<dyn Error>, line_number: u32, command: &str) -> Self {
         Self {
             inner: error,
-            filename,
-            line_number,
+            line_number: Some(line_number),
             command: command.to_string(),
         }
     }
-
-    pub fn from_outside(error: Box<dyn Error>, reason: &str) -> Self {
-        Self {
-            inner: error,
-            filename: "".to_string(),
-            line_number: 0,
-            command: reason.to_string(),
-        }
-    }
 }
 
 pub struct AssertReport {
+    filename: String,
     results: Vec<Result<WastSuccess, WastError>>,
 }
 
 impl AssertReport {
-    pub fn new() -> Self {
+    pub fn new(filename: &str) -> Self {
         Self {
+            filename: filename.to_string(),
             results: Vec::new(),
         }
     }
 
-    pub fn push_success(&mut self, filename: String, line_number: u32, command: String) {
-        self.results.push(Ok(WastSuccess {
-            filename,
-            line_number,
-            command,
-        }));
+    pub fn push_success(&mut self, success: WastSuccess) {
+        self.results.push(Ok(success));
     }
 
-    pub fn push_error(
-        &mut self,
-        filename: String,
-        line_number: u32,
-        command: String,
-        error: Box<dyn Error>,
-    ) {
-        self.results.push(Err(WastError {
-            inner: error,
-            filename,
-            line_number,
-            command,
-        }));
+    pub fn push_error(&mut self, error: WastError) {
+        self.results.push(Err(error));
     }
-}
 
-pub enum WastTestReport {
-    Asserts(AssertReport),
-    CompilationError(WastError),
-}
-
-impl From<WastError> for WastTestReport {
-    fn from(error: WastError) -> Self {
-        WastTestReport::CompilationError(error)
+    pub fn compile_report(self) -> WastTestReport {
+        return WastTestReport::Asserts(self);
     }
-}
 
-impl From<AssertReport> for WastTestReport {
-    fn from(assert_report: AssertReport) -> Self {
-        WastTestReport::Asserts(assert_report)
+    pub fn has_errors(&self) -> bool {
+        self.results.iter().any(|r| r.is_err())
     }
 }
 
-// .------------------------.
-// | Display Implementation |
-// '------------------------'
-
-impl std::fmt::Display for WastSuccess {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "[SUCCESS] {} ({}:{})",
-            self.command, self.filename, self.line_number
-        )
-    }
+pub struct CompilationError {
+    inner: Box<dyn Error>,
+    filename: String,
+    context: String,
 }
 
-impl std::fmt::Display for WastError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        writeln!(
-            f,
-            "[ERROR] {} ({}:{})",
-            self.command, self.filename, self.line_number
-        )?;
-        write!(f, "\t{}", self.inner)?;
+impl CompilationError {
+    pub fn new(error: Box<dyn Error>, filename: &str, context: &str) -> Self {
+        Self {
+            inner: error,
+            filename: filename.to_string(),
+            context: context.to_string(),
+        }
+    }
 
-        Ok(())
+    pub fn compile_report(self) -> WastTestReport {
+        return WastTestReport::CompilationError(self);
     }
 }
 
-impl std::fmt::Display for AssertReport {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        for result in &self.results {
-            match result {
-                Ok(success) => writeln!(f, "{}", success)?,
-                Err(error) => writeln!(f, "{}", error)?,
-            }
-        }
-
-        Ok(())
-    }
+pub enum WastTestReport {
+    Asserts(AssertReport),
+    CompilationError(CompilationError),
 }
 
+// .------------------------.
+// | Display Implementation |
+// '------------------------'
+
 impl std::fmt::Display for WastTestReport {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            WastTestReport::Asserts(assert_report) => write!(f, "{}", assert_report),
-            WastTestReport::CompilationError(error) => write!(f, "{}", error),
+            WastTestReport::CompilationError(error) => {
+                writeln!(f, "------ {} ------", error.filename)?;
+                writeln!(f, "⚠ Compilation Failed ⚠")?;
+                writeln!(f, "Context: {}", error.context)?;
+                writeln!(f, "Error: {}", error.inner)?;
+                writeln!(f, "~~~~~~~~~~~~~~~~")?;
+                writeln!(f, "")?;
+            }
+            WastTestReport::Asserts(assert_report) => {
+                writeln!(f, "------ {} ------", assert_report.filename)?;
+                for result in &assert_report.results {
+                    match result {
+                        Ok(success) => {
+                            writeln!(
+                                f,
+                                "✅ {}:{} -> {}",
+                                assert_report.filename, success.line_number, success.command
+                            )?;
+                        }
+                        Err(error) => {
+                            writeln!(
+                                f,
+                                "❌ {}:{} -> {}",
+                                assert_report.filename,
+                                error.line_number.unwrap_or(0),
+                                error.command
+                            )?;
+                            writeln!(f, "    Error: {}", error.inner)?;
+                        }
+                    }
+                }
+                let passed_asserts = assert_report.results.iter().filter(|r| r.is_ok()).count();
+                let failed_asserts = assert_report.results.iter().filter(|r| r.is_err()).count();
+                let total_asserts = assert_report.results.len();
+
+                writeln!(f, "")?;
+                writeln!(
+                    f,
+                    "Execution finished. Passed: {}, Failed: {}, Total: {}",
+                    passed_asserts, failed_asserts, total_asserts
+                )?;
+                writeln!(f, "~~~~~~~~~~~~~~~~")?;
+                writeln!(f, "")?;
+            }
         }
+
+        Ok(())
     }
 }