Python engine adapters

.github/workflows/surrealml_core_deployment.yml

@@ -4,6 +4,10 @@ on:
   push:
     branches:
       - main
+    paths:
+      - 'src/**'
+      - 'build.rs'
+      - 'Cargo.toml'
 
 jobs:
   post_merge_job:
@@ -15,6 +19,6 @@ jobs:
 
     - uses: katyo/publish-crates@v2
       with:
-          path: './modules/utils'
+          path: './modules/core'
           args: --no-verify
           registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/surrealml_core_test.yml

@@ -1,4 +1,4 @@
-name: Rust Test for surrealml-core on Pull Request
+name: Run tests on Pull Request
 
 on:
   pull_request:
@@ -20,5 +20,31 @@ jobs:
         toolchain: stable
         override: true
 
-    - name: Run Unit Tests
-      run: cd modules/utils && cargo test
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x' # Replace '3.x' with the specific version if needed
+
+    - name: Pre-test Setup
+      run: |
+        python3 -m venv venv
+        source venv/bin/activate
+        pip install -r requirements.txt
+
+        # build the local version of the core module to be loaded into python
+        python tests/scripts/local_build.py
+
+        # train the models for the tests
+        python tests/scripts/build_assets.py
+
+    - name: Run Python Unit Tests
+      run: |
+        source venv/bin/activate
+        python -m unittest discover
+        deactivate
+
+    - name: Run Core Unit Tests
+      run: cd modules/core && cargo test
+
+    - name: Run HTTP Transfer Tests
+      run: cargo test

.gitignore

@@ -6,13 +6,18 @@ Cargo.lock
 
 build/
 dist/
+__pycache__/
 venv/
 .idea/
 surrealml.egg-info/
 .vscode/
 ./modules/utils/target/
-modules/utils/target/
+modules/core/target/
 ./modules/onnx_driver/target/
 modules/onnx_driver/target/
 surrealdb_build/
-modules/utils/onnx_driver/
+modules/core/onnx_driver/
+*.so
+surrealml/rust_surrealml.cpython-310-darwin.so
+.surmlcache
+modules/core/model_stash/

Cargo.toml

@@ -16,7 +16,7 @@ ndarray = "0.15.6"
 hyper = { version = "0.14.27", features = ["full"] }
 tokio = { version = "1.34.0", features = ["full"] }
 base64 = "0.13"
-surrealml-core = { path = "./modules/utils" }
+surrealml-core = { path = "./modules/core" }
 
 [dev-dependencies]
 axum = "0.6.20"

build.rs

@@ -22,16 +22,16 @@ fn main() {
 
 	// remove ./modules/utils/target folder if there
 	let _ =
-		std::fs::remove_dir_all(Path::new("modules").join("utils").join("target")).unwrap_or(());
+		std::fs::remove_dir_all(Path::new("modules").join("core").join("target")).unwrap_or(());
 
 	// create the target module folder for the utils module
-	let _ = std::fs::create_dir(Path::new("modules").join("utils").join("target"));
-	let _ = std::fs::create_dir(Path::new("modules").join("utils").join("target").join(profile));
+	let _ = std::fs::create_dir(Path::new("modules").join("core").join("target"));
+	let _ = std::fs::create_dir(Path::new("modules").join("core").join("target").join(profile));
 
 	// copy target folder to modules/utils/target profile for the utils modules
 	std::fs::copy(
 		Path::new("target").join(profile).join(target_lib),
-		Path::new("modules").join("utils").join("target").join(profile).join(target_lib),
+		Path::new("modules").join("core").join("target").join(profile).join(target_lib),
 	)
 	.unwrap();
 }

modules/utils/Cargo.toml → modules/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "surrealml-core"
-version = "0.0.6"
+version = "0.0.8"
 edition = "2021"
 build = "./build.rs"
 description = "The core machine learning library for SurrealML that enables SurrealDB to store and load ML models"

modules/utils/src/execution/compute.rs → modules/core/src/execution/compute.rs

@@ -2,7 +2,7 @@
 use crate::storage::surml_file::SurMlFile;
 use std::collections::HashMap;
 use ndarray::{ArrayD, CowArray};
-use ort::{SessionBuilder, Value};
+use ort::{SessionBuilder, Value, session::Input};
 use super::onnx_environment::ENVIRONMENT;
 
 
@@ -29,6 +29,24 @@ impl <'a>ModelComputation<'a> {
         ndarray::arr1::<f32>(&buffer).into_dyn()
     }
 
+    /// Creates a vector of dimensions for the input tensor from the loaded model.
+    /// 
+    /// # Arguments
+    /// * `input_dims` - The input dimensions from the loaded model.
+    /// 
+    /// # Returns
+    /// A vector of dimensions for the input tensor to be reshaped into from the loaded model.
+    fn process_input_dims(input_dims: &Input) -> Vec<usize> {
+        let mut buffer = Vec::new();
+        for dim in input_dims.dimensions() {
+            match dim {
+                Some(dim) => buffer.push(dim as usize),
+                None => buffer.push(1)
+            }
+        }
+        buffer
+    }
+
     /// Creates a Vector that can be used manipulated with other operations such as normalisation from a hashmap of keys and values.
     /// 
     /// # Arguments
@@ -52,28 +70,14 @@ impl <'a>ModelComputation<'a> {
     /// 
     /// # Returns
     /// The computed output tensor from the loaded model.
-    pub fn raw_compute(&self, tensor: ArrayD<f32>, dims: Option<(i32, i32)>) -> Result<Vec<f32>, String> {
-
-        let tensor_placeholder: ArrayD<f32>;
-        if dims.is_some() {
-            let dims = dims.unwrap();
-            let tensor = tensor.into_shape((dims.0 as usize, dims.1 as usize)).unwrap();
-            tensor_placeholder = tensor.into_dyn();
-        }
-        else {
-            tensor_placeholder = tensor;
-        }
-
-        // let environment = Arc::new(
-        //     Environment::builder()
-        //         .with_execution_providers([ExecutionProvider::CPU(Default::default())])
-        //         .build()
-        //         .map_err(|e| e.to_string())?
-        // );
+    pub fn raw_compute(&self, tensor: ArrayD<f32>, _dims: Option<(i32, i32)>) -> Result<Vec<f32>, String> {
         let session = SessionBuilder::new(&ENVIRONMENT).map_err(|e| e.to_string())?
                                                        .with_model_from_memory(&self.surml_file.model)
                                                        .map_err(|e| e.to_string())?;
-        let x = CowArray::from(tensor_placeholder);
+        let unwrapped_dims = ModelComputation::process_input_dims(&session.inputs[0]);
+        let tensor = tensor.into_shape(unwrapped_dims).map_err(|e| e.to_string())?;
+
+        let x = CowArray::from(tensor).into_dyn();
         let outputs = session.run(vec![Value::from_array(session.allocator(), &x).unwrap()]).map_err(|e| e.to_string())?;
 
         let mut buffer: Vec<f32> = Vec::new();
@@ -148,9 +152,8 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_raw_compute() {
-
-        let mut file = SurMlFile::from_file("./stash/test.surml").unwrap();
+    fn test_raw_compute_linear_sklearn() {
+        let mut file = SurMlFile::from_file("./model_stash/sklearn/surml/linear.surml").unwrap();
         let model_computation = ModelComputation {
             surml_file: &mut file,
         };
@@ -159,15 +162,17 @@ mod tests {
         input_values.insert(String::from("squarefoot"), 1000.0);
         input_values.insert(String::from("num_floors"), 2.0);
 
-        let output = model_computation.raw_compute(model_computation.input_tensor_from_key_bindings(input_values), None).unwrap();
+        let raw_input = model_computation.input_tensor_from_key_bindings(input_values);
+
+        let output = model_computation.raw_compute(raw_input, Some((1, 2))).unwrap();
         assert_eq!(output.len(), 1);
-        assert_eq!(output[0], 725.42053);
+        assert_eq!(output[0], 985.57745);
     }
 
 
     #[test]
-    fn test_buffered_compute() {
-        let mut file = SurMlFile::from_file("./stash/test.surml").unwrap();
+    fn test_buffered_compute_linear_sklearn() {
+        let mut file = SurMlFile::from_file("./model_stash/sklearn/surml/linear.surml").unwrap();
         let model_computation = ModelComputation {
             surml_file: &mut file,
         };
@@ -178,22 +183,24 @@ mod tests {
 
         let output = model_computation.buffered_compute(&mut input_values).unwrap();
         assert_eq!(output.len(), 1);
-        assert_eq!(output[0], 725.42053);
+        assert_eq!(output[0], 985.57745);
     }
 
     #[test]
-    fn test_raw_compute_trees() {
-            let mut file = SurMlFile::from_file("./stash/forrest.surml").unwrap();
-            let model_computation = ModelComputation {
-                surml_file: &mut file,
-            };
-
-            let x = vec![0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1];
-            let data: ArrayD<f32> = ndarray::arr1(&x).into_dyn();
-            let data: ArrayD<f32> = data.into_shape((1, 28)).unwrap().into_dyn();
-
-            let output = model_computation.raw_compute(data, None).unwrap();
-            assert_eq!(output.len(), 1);
-            assert_eq!(output[0], 0.0);
+    fn test_raw_compute_linear_torch() {
+        let mut file = SurMlFile::from_file("./model_stash/torch/surml/linear.surml").unwrap();
+        let model_computation = ModelComputation {
+            surml_file: &mut file,
+        };
+
+        let mut input_values = HashMap::new();
+        input_values.insert(String::from("squarefoot"), 1000.0);
+        input_values.insert(String::from("num_floors"), 2.0);
+
+        let raw_input = model_computation.input_tensor_from_key_bindings(input_values);
+
+        let output = model_computation.raw_compute(raw_input, None).unwrap();
+        assert_eq!(output.len(), 1);
+        assert_eq!(output[0], 378.237);
     }
 }

requirements.txt

@@ -0,0 +1,4 @@
+numpy==1.26.3
+skl2onnx==1.16.0
+scikit-learn==1.4.0
+torch==2.1.2

scripts/local_build.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# navigate to directory
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $SCRIPTPATH
+
+
+delete_directory() {
+    dir_path="$1"
+
+    if [ -d "$dir_path" ]; then
+        rm -rf "$dir_path"
+        echo "Directory '$dir_path' has been deleted."
+    else
+        echo "Directory '$dir_path' does not exist."
+    fi
+}
+
+delete_file() {
+    file_path="$1"
+
+    if [ -f "$file_path" ]; then
+        rm "$file_path"
+        echo "File '$file_path' has been deleted."
+    else
+        echo "File '$file_path' does not exist."
+    fi
+}
+
+
+cd ..
+
+delete_directory ./build
+delete_directory ./tests/venv
+cd tests
+python3 -m venv venv
+source venv/bin/activate
+cd ..
+pip install --no-cache-dir .

scripts/run_tests.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# navigate to directory
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+cd $SCRIPTPATH
+
+cd ..
+
+
+python -m unittest discover

setup.py

@@ -18,7 +18,8 @@
     rust_extensions=[RustExtension("surrealml.rust_surrealml", binding=Binding.PyO3)],
     packages=[
         "surrealml",
-        # "surrealdb.execution_mixins"
+        "surrealml.engine",
+        "surrealml.model_templates"
     ],
     package_data={
         "surrealml": ["binaries/*"],
@@ -27,9 +28,15 @@
     zip_safe=False,
     include_package_data=True,
     requirements=[
-        "pyyaml>=3.13",
-        "numpy",
-        "torch==2.0.0",
-        "hummingbird-ml==0.4.9"
-    ]
+        "numpy==1.26.3",
+    ],
+    extras_require={
+        "sklearn": [
+            "skl2onnx==1.16.0",
+            "scikit-learn==1.4.0"
+        ],
+        "torch": [
+            "torch==2.1.2"
+        ]
+    }
 )