From 2551c9803299c4e4d321bc32aea3c0dfc5bcb093 Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Thu, 21 Nov 2024 11:34:34 +0800 Subject: [PATCH 01/12] refactor: unify the namespace of module/ inline printer Display the Solverz version used to generate modules --- Solverz/__init__.py | 2 +- .../python/module/module_generator.py | 23 ++++++------------- .../module/test/test_module_generator.py | 22 ++++++++---------- Solverz/num_api/module_parser.py | 21 ++++++++++++++--- docs/src/advanced.md | 1 - 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/Solverz/__init__.py b/Solverz/__init__.py index 83e5e4d..fe26925 100644 --- a/Solverz/__init__.py +++ b/Solverz/__init__.py @@ -3,7 +3,7 @@ from Solverz.equation.param import Param, IdxParam, TimeSeriesParam from Solverz.sym_algebra.symbols import idx, Para, iVar, iAliasVar from Solverz.sym_algebra.functions import (Sign, Abs, transpose, exp, Diag, Mat_Mul, sin, cos, Min, AntiWindUp, - Saturation, heaviside, ln) + Saturation, heaviside, ln, MulVarFunc, UniVarFunc) from Solverz.variable.variables import Vars, TimeVars, as_Vars from Solverz.solvers import * from Solverz.code_printer import made_numerical, module_printer diff --git a/Solverz/code_printer/python/module/module_generator.py b/Solverz/code_printer/python/module/module_generator.py index 120eebd..053d49b 100644 --- a/Solverz/code_printer/python/module/module_generator.py +++ b/Solverz/code_printer/python/module/module_generator.py @@ -67,8 +67,6 @@ def render_modules(eqs: SymEquations, code_dict["inner_Hvp"] = inner_Hvp['code_inner_Hvp'] code_dict["sub_inner_Hvp"] = inner_Hvp['code_sub_inner_Hvp'] - - def print_trigger_func_code(): code_tfuc = dict() trigger_func = parse_trigger_func(eqs.PARAM) @@ -176,7 +174,11 @@ def create_python_module(module_name, def print_init_code(eqn_type: str, module_name, eqn_param): - code = 'from .num_func import F_, J_\n' + code = '"""\n' + from ...._version import __version__ + code += f'Python module generated by Solverz {__version__}\n' + code += '"""\n' + code += 'from .num_func import F_, J_\n' code += 'from .dependency import setting, p_, y\n' code += 'import time\n' match eqn_type: @@ -207,7 +209,7 @@ def print_init_code(eqn_type: str, module_name, eqn_param): code += f'start = time.perf_counter()\n' code += code_compile.format(alpha=code_compile_args_F_J, beta=code_compile_args_Hvp) code += f'end = time.perf_counter()\n' - code += "print(f'Compiling time elapsed: {end-start}s')\n" + code += "print(f'Compiling time elapsed: {end - start}s')\n" return code @@ -264,25 +266,14 @@ def print_module_code(code_dict: Dict[str, str], numba=False): return code -# -code_from_SolMuseum=""" -try: - import SolMuseum.num_api as SolMF -except ImportError as e: - pass -""" def print_dependency_code(modules): code = "import os\n" code += "current_module_dir = os.path.dirname(os.path.abspath(__file__))\n" code += 'from Solverz import load\n' code += 'auxiliary = load(f"{current_module_dir}\\\\param_and_setting.pkl")\n' code += 'from numpy import *\n' - code += 'import numpy as np\n' - code += 'import Solverz.num_api.custom_function as SolCF\n' # import Solverz built-in func - code += code_from_SolMuseum - code += 'import scipy.sparse as sps\n' - code += 'from numba import njit\n' + code += 'from Solverz.num_api.module_parser import *\n' code += 'setting = auxiliary["eqn_param"]\n' code += 'row = setting["row"]\n' code += 'col = setting["col"]\n' diff --git a/Solverz/code_printer/python/module/test/test_module_generator.py b/Solverz/code_printer/python/module/test/test_module_generator.py index 0731404..1de3030 100644 --- a/Solverz/code_printer/python/module/test/test_module_generator.py +++ b/Solverz/code_printer/python/module/test/test_module_generator.py @@ -18,16 +18,7 @@ from Solverz import load auxiliary = load(f"{current_module_dir}\\param_and_setting.pkl") from numpy import * -import numpy as np -import Solverz.num_api.custom_function as SolCF - -try: - import SolMuseum.num_api as SolMF -except ImportError as e: - pass - -import scipy.sparse as sps -from numba import njit +from Solverz.num_api.module_parser import * setting = auxiliary["eqn_param"] row = setting["row"] col = setting["col"] @@ -38,7 +29,12 @@ y = auxiliary["vars"] """ -expected_init = r"""from .num_func import F_, J_ +from ....._version import __version__ + +expected_init = r'''""" +Python module generated by Solverz {vs} +""" +from .num_func import F_, J_ from .dependency import setting, p_, y import time from Solverz.num_api.num_eqn import nAE @@ -60,8 +56,8 @@ v = ones_like(y) mdl.HVP(y, p_, v) end = time.perf_counter() -print(f'Compiling time elapsed: {end-start}s') -""" +print(f'Compiling time elapsed: {{end - start}}s') +'''.format(vs=__version__) def test_AE_module_printer(): diff --git a/Solverz/num_api/module_parser.py b/Solverz/num_api/module_parser.py index 7c0fd44..199ca89 100644 --- a/Solverz/num_api/module_parser.py +++ b/Solverz/num_api/module_parser.py @@ -4,13 +4,28 @@ import warnings import Solverz.num_api.custom_function as SolCF +import numpy as np +import scipy.sparse as sps +from numba import njit -modules = [{'SolCF': SolCF, 'np': numpy, 'sps': scipy.sparse}, 'numpy'] -# We preserve the 'numpy' here in case one uses functions from sympy instead of from Solverz +module_dict = {'SolCF': SolCF, 'np': np, 'sps': sps, 'njit': njit} # parse modules from museum try: import SolMuseum.num_api as SolMF - modules[0]['SolMF'] = SolMF + module_dict['SolMF'] = SolMF except ModuleNotFoundError as e: warnings.warn(f'Failed to import num api from SolMuseum: {e}') + +# parse user defined functions +import sys +sys.path.extend(['D:\\OneDrive\\dev\\myfunc']) +try: + import myfunc + module_dict['myfunc'] = myfunc +except ModuleNotFoundError as e: + warnings.warn(f'Failed to import num api from myfunc: {e}') + +modules = [module_dict, 'numpy'] +# We preserve the 'numpy' here in case one uses functions from sympy instead of from Solverz +__all__ = list(module_dict.keys()) diff --git a/docs/src/advanced.md b/docs/src/advanced.md index 95e8227..6f9746e 100644 --- a/docs/src/advanced.md +++ b/docs/src/advanced.md @@ -124,7 +124,6 @@ It is recommended that one first gets familiar with the [numpy](https://numpy.or ``` The implementation of `SolLessThan()` should be put in the `Solverz.num_api.custom_function` module: ```python -@implements_nfunc('SolLessThan') @njit(cache=True) def SolLessThan(x, y): x = np.asarray(x).reshape((-1,)) From 939842f3034aab60ef8ada102d375f939151c7e4 Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Thu, 21 Nov 2024 11:57:43 +0800 Subject: [PATCH 02/12] test: use dev mode to perform tests --- .github/workflows/run-tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0ac5c16..cbec3c3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -19,9 +19,6 @@ jobs: python-version: '3.11' cache: 'pip' # caching pip dependencies - run: | - pip install -r requirements.txt - cd docs/ - pip install -r requirements.txt - cd .. + pip install -e . - run: | # run both independent pytest and doctest pytest From 2fcbf211c3a71124922b179a53721c5aeed179d1 Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Thu, 21 Nov 2024 12:05:27 +0800 Subject: [PATCH 03/12] test: accelerate cookbook tests using pytest-xdist --- .github/workflows/cookbook-practice.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cookbook-practice.yml b/.github/workflows/cookbook-practice.yml index a173140..906c266 100644 --- a/.github/workflows/cookbook-practice.yml +++ b/.github/workflows/cookbook-practice.yml @@ -27,7 +27,8 @@ jobs: run: | pip install --upgrade pip setuptools wheel pip install git+https://github.com/${{github.repository}}.git@${{ github.sha }} + pip install pytest-xdist - name: Run Tests run: | - pytest + pytest -n auto From ca90646561cf83ec0184daa4b556b28639ffe138 Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Thu, 21 Nov 2024 17:04:49 +0800 Subject: [PATCH 04/12] feat: enable setting of UDF path User defined function --- Solverz/__init__.py | 1 + Solverz/num_api/module_parser.py | 21 ++-- Solverz/num_api/test/test_my_module_parser.py | 0 Solverz/num_api/user_function_parser.py | 116 ++++++++++++++++++ 4 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 Solverz/num_api/test/test_my_module_parser.py create mode 100644 Solverz/num_api/user_function_parser.py diff --git a/Solverz/__init__.py b/Solverz/__init__.py index fe26925..df6df46 100644 --- a/Solverz/__init__.py +++ b/Solverz/__init__.py @@ -11,6 +11,7 @@ from Solverz.utilities.profile import count_time from Solverz.variable.ssymbol import Var, AliasVar from Solverz.model.basic import Model +from Solverz.num_api.user_function_parser import save_module_paths, reset_module_paths from importlib.metadata import version, PackageNotFoundError diff --git a/Solverz/num_api/module_parser.py b/Solverz/num_api/module_parser.py index 199ca89..f2f6534 100644 --- a/Solverz/num_api/module_parser.py +++ b/Solverz/num_api/module_parser.py @@ -18,13 +18,20 @@ warnings.warn(f'Failed to import num api from SolMuseum: {e}') # parse user defined functions -import sys -sys.path.extend(['D:\\OneDrive\\dev\\myfunc']) -try: - import myfunc - module_dict['myfunc'] = myfunc -except ModuleNotFoundError as e: - warnings.warn(f'Failed to import num api from myfunc: {e}') +from .user_function_parser import load_module_paths + +user_module_paths = load_module_paths() +if user_module_paths: + print('User module detected.') + import os, sys + for path in user_module_paths: + module_name = os.path.splitext(os.path.basename(path))[0] + module_dir = os.path.dirname(path) + + sys.path.insert(0, module_dir) + exec('import ' + module_name) + module_dict[module_name] = globals()[module_name] + modules = [module_dict, 'numpy'] # We preserve the 'numpy' here in case one uses functions from sympy instead of from Solverz diff --git a/Solverz/num_api/test/test_my_module_parser.py b/Solverz/num_api/test/test_my_module_parser.py new file mode 100644 index 0000000..e69de29 diff --git a/Solverz/num_api/user_function_parser.py b/Solverz/num_api/user_function_parser.py new file mode 100644 index 0000000..e262234 --- /dev/null +++ b/Solverz/num_api/user_function_parser.py @@ -0,0 +1,116 @@ +import os +from pathlib import Path +import sys +import importlib.util + + +def validate_module_paths(paths): + """ + Validate that each path in the provided list points to a valid Python module. + + :param paths: List of paths + :return: List of validated paths + :raises ValueError: If a path is invalid or not a Python module + """ + valid_paths = [] + for path in paths: + # Check if the path exists + if not os.path.exists(path): + raise ValueError(f"The path {path} does not exist.") + + # Check if the path is a file + if not os.path.isfile(path): + raise ValueError(f"The path {path} is not a file.") + + # Check if the file is a Python file + if not path.endswith('.py'): + raise ValueError(f"The file {path} is not a Python file.") + + # If all checks pass, add the path to the valid paths list + valid_paths.append(path) + + return valid_paths + + +def save_module_paths(paths, filename='user_modules.txt'): + """ + Save user-provided module paths to a specified file, but validate the paths before saving. + If a path already exists in the file, it will not be added again. + + :param paths: List of user-provided module paths + :param filename: Name of the file to save, default is 'user_modules.txt' + """ + try: + # Validate paths + validated_paths = validate_module_paths(paths) + except ValueError as e: + print(e) + return + + # Get the path to the .Solverz directory in the user's home directory + user_home = str(Path.home()) + solverz_dir = os.path.join(user_home, '.Solverz') + + # Create the .Solverz directory if it does not exist + if not os.path.exists(solverz_dir): + os.makedirs(solverz_dir) + + # Define the full path to the file + file_path = os.path.join(solverz_dir, filename) + + # Read existing paths from the file + existing_paths = set() + if os.path.exists(file_path): + with open(file_path, 'r') as file: + existing_paths = set(line.strip() for line in file) + + # Write the new paths to the file, but only if they are not already present + with open(file_path, 'a') as file: + for path in validated_paths: + if path not in existing_paths: + file.write(f'{path}\n') + existing_paths.add(path) + + +def load_module_paths(filename='user_modules.txt'): + """ + Load module paths from a specified file in the .Solverz directory in the user's home directory. + + :param filename: Name of the file to load, default is 'user_modules.txt' + :return: List of module paths + """ + user_home = str(Path.home()) + solverz_dir = os.path.join(user_home, '.Solverz') + file_path = os.path.join(solverz_dir, filename) + + # Check if the file exists + if not os.path.exists(file_path): + return [] + + # Read and return the list of paths + with open(file_path, 'r') as file: + paths = [line.strip() for line in file] + + return paths + + +def reset_module_paths(filename='user_modules.txt'): + """ + Reset the user_modules.txt file by clearing its content. + + :param filename: Name of the file to reset, default is 'user_modules.txt' + """ + # Get the path to the .Solverz directory in the user's home directory + user_home = str(Path.home()) + solverz_dir = os.path.join(user_home, '.Solverz') + + # Define the full path to the file + file_path = os.path.join(solverz_dir, filename) + + # Create the .Solverz directory if it does not exist + if not os.path.exists(solverz_dir): + os.makedirs(solverz_dir) + + # Clear the content of the file + with open(file_path, 'w') as file: + file.write('') From 9d4c1764bb54296a96fc7a125683491c05cc708d Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Thu, 21 Nov 2024 20:21:42 +0800 Subject: [PATCH 05/12] refactor: rename user_function_parser --- Solverz/__init__.py | 2 +- Solverz/num_api/module_parser.py | 4 ++-- Solverz/num_api/user_function_parser.py | 8 +++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Solverz/__init__.py b/Solverz/__init__.py index df6df46..1f43a57 100644 --- a/Solverz/__init__.py +++ b/Solverz/__init__.py @@ -11,7 +11,7 @@ from Solverz.utilities.profile import count_time from Solverz.variable.ssymbol import Var, AliasVar from Solverz.model.basic import Model -from Solverz.num_api.user_function_parser import save_module_paths, reset_module_paths +from Solverz.num_api.user_function_parser import add_my_module, reset_my_module_paths from importlib.metadata import version, PackageNotFoundError diff --git a/Solverz/num_api/module_parser.py b/Solverz/num_api/module_parser.py index f2f6534..86709d6 100644 --- a/Solverz/num_api/module_parser.py +++ b/Solverz/num_api/module_parser.py @@ -18,9 +18,9 @@ warnings.warn(f'Failed to import num api from SolMuseum: {e}') # parse user defined functions -from .user_function_parser import load_module_paths +from .user_function_parser import load_my_module_paths -user_module_paths = load_module_paths() +user_module_paths = load_my_module_paths() if user_module_paths: print('User module detected.') import os, sys diff --git a/Solverz/num_api/user_function_parser.py b/Solverz/num_api/user_function_parser.py index e262234..247ccd3 100644 --- a/Solverz/num_api/user_function_parser.py +++ b/Solverz/num_api/user_function_parser.py @@ -1,7 +1,5 @@ import os from pathlib import Path -import sys -import importlib.util def validate_module_paths(paths): @@ -32,7 +30,7 @@ def validate_module_paths(paths): return valid_paths -def save_module_paths(paths, filename='user_modules.txt'): +def add_my_module(paths, filename='user_modules.txt'): """ Save user-provided module paths to a specified file, but validate the paths before saving. If a path already exists in the file, it will not be added again. @@ -72,7 +70,7 @@ def save_module_paths(paths, filename='user_modules.txt'): existing_paths.add(path) -def load_module_paths(filename='user_modules.txt'): +def load_my_module_paths(filename='user_modules.txt'): """ Load module paths from a specified file in the .Solverz directory in the user's home directory. @@ -94,7 +92,7 @@ def load_module_paths(filename='user_modules.txt'): return paths -def reset_module_paths(filename='user_modules.txt'): +def reset_my_module_paths(filename='user_modules.txt'): """ Reset the user_modules.txt file by clearing its content. From 50f28aad9f76c170808c12684f25a842b4ffd5ca Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Thu, 21 Nov 2024 22:13:02 +0800 Subject: [PATCH 06/12] test: add UDM test --- Solverz/num_api/test/test_my_module_parser.py | 0 Solverz/num_api/test/test_udm.py | 75 +++++++++++++++++++ Solverz/num_api/user_function_parser.py | 21 +++--- 3 files changed, 84 insertions(+), 12 deletions(-) delete mode 100644 Solverz/num_api/test/test_my_module_parser.py create mode 100644 Solverz/num_api/test/test_udm.py diff --git a/Solverz/num_api/test/test_my_module_parser.py b/Solverz/num_api/test/test_my_module_parser.py deleted file mode 100644 index e69de29..0000000 diff --git a/Solverz/num_api/test/test_udm.py b/Solverz/num_api/test/test_udm.py new file mode 100644 index 0000000..b713401 --- /dev/null +++ b/Solverz/num_api/test/test_udm.py @@ -0,0 +1,75 @@ +""" +Test the user defined modules. +""" + +import importlib +import os +import re +import shutil +from pathlib import Path + +import pytest + +from Solverz.num_api.user_function_parser import add_my_module, reset_my_module_paths + +mymodule_code = """import numpy as np +from numba import njit + + +@njit(cache=True) +def c(x, y): + x = np.asarray(x).reshape((-1,)) + y = np.asarray(y).reshape((-1,)) + + z = np.zeros_like(x) + + for i in range(len(x)): + if x[i] <= y[i]: + z[i] = x[i] + else: + z[i] = y[i] + return z +""" + + +def test_udm(): + # Create a .Solverz_test_temp directory in the user's home directory + user_home = str(Path.home()) + solverz_dir = os.path.join(user_home, '.Solverz_test_temp') + + # Create the .Solverz directory if it does not exist + if not os.path.exists(solverz_dir): + os.makedirs(solverz_dir) + + file_path = os.path.join(solverz_dir, r'your_module.py') + file_path1 = os.path.join(solverz_dir, r'fake1.jl') + + # Write the new paths to the file, but only if they are not already present + with open(file_path, 'a') as file: + file.write(mymodule_code) + + with open(file_path1, 'a') as file: + file.write(mymodule_code) + + with pytest.raises(ValueError, + match=re.escape(f"The path {solverz_dir} is not a file.")): + add_my_module([solverz_dir]) + + with pytest.raises(ValueError, + match=re.escape(f"The path {os.path.join(user_home, '.Solverz_test_temp1')} does not exist.")): + add_my_module([os.path.join(user_home, '.Solverz_test_temp1')]) + + with pytest.raises(ValueError, + match=re.escape(f"The file {file_path1} is not a Python file.")): + add_my_module([file_path1]) + + add_my_module([file_path]) + + import Solverz + importlib.reload(Solverz.num_api.module_parser) + from Solverz.num_api.module_parser import your_module + import numpy as np + np.testing.assert_allclose(your_module.c(np.array([1, 0]), np.array([2, -1])), np.array([1, -1])) + + shutil.rmtree(solverz_dir) + reset_my_module_paths() diff --git a/Solverz/num_api/user_function_parser.py b/Solverz/num_api/user_function_parser.py index 247ccd3..f10c5df 100644 --- a/Solverz/num_api/user_function_parser.py +++ b/Solverz/num_api/user_function_parser.py @@ -16,13 +16,13 @@ def validate_module_paths(paths): if not os.path.exists(path): raise ValueError(f"The path {path} does not exist.") - # Check if the path is a file - if not os.path.isfile(path): - raise ValueError(f"The path {path} is not a file.") + # Check if the path is a file + if not os.path.isfile(path): + raise ValueError(f"The path {path} is not a file.") - # Check if the file is a Python file - if not path.endswith('.py'): - raise ValueError(f"The file {path} is not a Python file.") + # Check if the file is a Python file + if not path.endswith('.py'): + raise ValueError(f"The file {path} is not a Python file.") # If all checks pass, add the path to the valid paths list valid_paths.append(path) @@ -38,12 +38,9 @@ def add_my_module(paths, filename='user_modules.txt'): :param paths: List of user-provided module paths :param filename: Name of the file to save, default is 'user_modules.txt' """ - try: - # Validate paths - validated_paths = validate_module_paths(paths) - except ValueError as e: - print(e) - return + + # Validate paths + validated_paths = validate_module_paths(paths) # Get the path to the .Solverz directory in the user's home directory user_home = str(Path.home()) From f7702df6df37eaa82cfc120e98852081206df399 Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Fri, 22 Nov 2024 12:16:55 +0800 Subject: [PATCH 07/12] docs: change the advanced usage. --- docs/src/advanced.md | 235 +++++++++++++++++++++++++++---------------- 1 file changed, 147 insertions(+), 88 deletions(-) diff --git a/docs/src/advanced.md b/docs/src/advanced.md index 6f9746e..ee629bd 100644 --- a/docs/src/advanced.md +++ b/docs/src/advanced.md @@ -4,130 +4,189 @@ ## Writing Custom Functions Sometimes one may have functions that go beyond the Solverz built-in library. This guide will describe how to create -such custom functions in Solverz, so that the functions can be incorporated into numerical simulations. The philosophy -of function customization comes from Sympy, it helps to learn the [Sympy basics](https://docs.sympy.org/latest/index.html) +such custom functions using Solverz and inform Solverz of their paths, so that the functions can be incorporated into +numerical simulations. + +```{note} +Alternatively, one can directly contribute to the [SolMuseum](https://solmuseum.solverz.org/stable/) +library so that 1) others can utilize your models/functions and 2) one avoids configuring the module paths. +``` + +```{note} +The philosophy of function customization comes from Sympy, it helps to learn the [Sympy basics](https://docs.sympy.org/latest/index.html) and read the [Sympy tutorial of custom functions](https://docs.sympy.org/latest/guides/custom-functions.html) for an overview. +``` + +```{note} +In Solverz, the numerical computations are mainly dependent on the prevailing numerical libraries such as numpy and scipy. +It is recommended that one first gets familiar with the [numpy](https://numpy.org/doc/stable/user/index.html) and +[scipy](https://docs.scipy.org/doc/scipy/tutorial/index.html). +``` -As a motivating example for this document, let's create a custom function class representing the $\min$ function. We -want use $\min$ to determine the smaller one of two operands, which can be defined by +### An Illustrative Example + +As a motivating example for this document, let's create a custom function class representing the $\min$ function. +The $\min$ function is typical in controllers of many industrial applications, which can be defined by ```{math} \min(x,y)= \begin{cases} x&x\leq y\\ y& x>y -\end{cases} +\end{cases}. ``` -We also want to extend the function to vector input, that is, capable of finding the element-wise minimum of two vectors. -To summarize, we shall implement $\min$ that +To incorporate $\min$ in our simulation modelling, its symbolic and numerical implementations shall be defined. +Specifically, -1. evaluates $\min(x,y)$ correctly -2. can be derived proper derivatives with respect to $x$ and $y$. +1. a symbolic function `min` can be called to represent the $\min$ function; +2. the symbolic derivatives of `min` are automatically derived for the Jacobian block parser; +3. the numerical interfaces are defined so that the `min` function and its derivatives can be correctly evaluated. -However, it is difficult to devise the analytical derivatives of $\min$. We should perform the trick that rewrites -$\min(x,y)$ as +First, we define the numerical interfaces. The derivatives of $\min$ function are ```{math} -x*\operatorname{lessthan}(x,y)+y*(1-\operatorname{lessthan}(x,y)). +\pdv{\min(x,y)}{x}= +\begin{cases} +1&x\leq y\\ +0& x>y +\end{cases},\quad +\pdv{\min(x,y)}{y}= +\begin{cases} +0&x\leq y\\ +1& x>y +\end{cases}. ``` -where the $\operatorname{lessthan}(x,y)$ function mathematically denotes the $\leq$ operator and returns 1 if -$x\leq y$ else 0. Since $\operatorname{lessthan}(x,y)$ can only be either 1 or 0, the above transformation holds. +Let us create a `myfunc.py` file and put the numerical codes there, which will look like -If the derivatives of $\operatorname{lessthan}(x,y)$ with respect to any argument are zero, then we have -```{math} -\frac{\partial}{\partial x}\min(x,y)= -\operatorname{lessthan}(x,y) +```python +# myfunc.py +import numpy as np +from numba import njit + + +@njit(cache=True) +def Min(x, y): + x = np.asarray(x).reshape((-1,)) + y = np.asarray(y).reshape((-1,)) + + z = np.zeros_like(x) + + for i in range(len(x)): + if x[i] <= y[i]: + z[i] = x[i] + else: + z[i] = y[i] + return z + + +@njit(cache=True) +def dMindx(x, y): + x = np.asarray(x).reshape((-1,)) + y = np.asarray(y).reshape((-1,)) + + z = np.zeros_like(x) + + for i in range(len(x)): + if x[i] <= y[i]: + z[i] = 1 + else: + z[i] = 0 + return z + + +@njit(cache=True) +def dMindy(x, y): + return 1-dMindx(x, y) ``` -and -```{math} -\frac{\partial}{\partial y}\min(x,y)= -1-\operatorname{lessthan}(x,y). + +where we use `Min` to avoid conflicting with the built-in `min` function, +the `@njit(cache)` decorator is used to perform the jit-compilation and hence speed up the numerical codes. + +Then let us add the path of `myfunc.py` to Solverz, with, for example, + +```python +from Solverz import add_my_module +add_my_module([r'D:\myfunc.py']) ``` -Hence, it suffices to have a custom $\operatorname{lessthan}(x,y)$ function that -1. evaluates $\operatorname{lessthan}(x,y)$ correctly -2. has zero-derivative with respect to $x$ or $y$. +Then Solverz would load your functions defined in the `myfunc` module. One can also reset the paths by calling +`reset_my_module_paths`. + +As for the symbolic implementation, let us start by creating a `Min.py` file and subclassing `MulVarFunc` there with -Let us start by subclassing `MulVarFunc` ```python -from Solverz.sym_algebra.functions import MulVarFunc +from Solverz import MulVarFunc + class Min(MulVarFunc): pass -class LessThan(MulVarFunc): + +class dMindx(MulVarFunc): + pass + +class dMindy(MulVarFunc): pass ``` -The `MulVarFunc` is the base class of multi-variate functions in Solverz. -At this point, `Min` has no behaviors defined on it. To automatically evaluate the `Min` function, we ought to define -the **_class method_** `eval()`. `eval()` should take the arguments of the function and return the value -$x*\operatorname{lessthan}(x,y)+y*(1-\operatorname{lessthan}(x,y))$: + +The `MulVarFunc` is the base class of symbolic multi-variate functions in Solverz. + +At this point, `Min` and its derivatives have no behaviors defined on it. To instruct Solverz in the differentiation +rule of `Min` and the numerical implementations, we shall add following lines ```python class Min(MulVarFunc): - @classmethod - def eval(cls, x, y): - return x * LessThan(x, y) + y * (1 - LessThan(x, y)) -``` -```python ->>> from Solverz import Var ->>> Min(Var('x',0),Var('y',0)) -... x*((x)<=(y)) + y*(1 - ((x)<=(y))) -``` -To define the differentiation of `LessThan()`, we have -```python -from sympy import Integer -class LessThan(MulVarFunc): - """ - Represents < operator - """ + arglength = 2 - def _eval_derivative(self, s): - return Integer(0) + def fdiff(self, argindex=1): + if argindex == 1: + return dMindx(*self.args) + elif argindex == 2: + return dMindy(*self.args) + + def _numpycode(self, printer, **kwargs): + return (f'myfunc.Min' + r'(' + + ', '.join([printer._print(arg, **kwargs) for arg in self.args]) + r')') - def _sympystr(self, printer, **kwargs): - return '(({op1})<=({op2}))'.format(op1=printer._print(self.args[0]), - op2=printer._print(self.args[1])) + +class dMindx(MulVarFunc): + arglength = 2 def _numpycode(self, printer, **kwargs): - return r'SolLessThan(' + ', '.join([printer._print(arg, **kwargs) for arg in self.args]) + r')' + return (f'myfunc.dMindx' + r'(' + + ', '.join([printer._print(arg, **kwargs) for arg in self.args]) + r')') - def _lambdacode(self, printer, **kwargs): - return self._numpycode(printer, **kwargs) - def _pythoncode(self, printer, **kwargs): - return self._numpycode(printer, **kwargs) -``` -Here, the `_sympystr()` method defines its string representation: -```python ->>> LessThan(Var('x'), Var('y')) -... ((x)<=(y)) -``` -The `_eval_derivative()` method forces the derivatives of `LessThan()` to be zero: -```python -from Solverz import iVar ->>> Min(Var('x',0),Var('y',0)).diff(iVar('x')) -... ((x)<=(y)) -``` -where `iVar` is the internal variable type of Solverz, `diff()` is the method to derive derivatives. +class dMindy(MulVarFunc): + arglength = 2 + + def _numpycode(self, printer, **kwargs): + return (f'myfunc.dMindy' + r'(' + + ', '.join([printer._print(arg, **kwargs) for arg in self.args]) + r')') -The `_numpycode()` function defines what should `LessThan()` be printed to in numerical codes. Here, we define the -`SolLessThan()` as the numerical implementation of `LessThan()`. Given array `[0,2,-1]` and `[1,2,3]`: -```python ->>> import numpy as np ->>> SolLessThan(np.array([0, 2, -1]), np.array([1,2,3])) -... array([1, 0, 1]) -``` -```{note} -In Solverz, the numerical computations are mainly dependent on the prevailing numerical libraries such as numpy and scipy. -It is recommended that one first gets familiar with the [numpy](https://numpy.org/doc/stable/user/index.html) and -[scipy](https://docs.scipy.org/doc/scipy/tutorial/index.html). ``` -The implementation of `SolLessThan()` should be put in the `Solverz.num_api.custom_function` module: + +where the `fdiff` function should return the derivative of the function, without considering the chain rule, +with respect to the `argindex`-th variable; the `_numpycode` functions define the numerical implementations of the +functions. Since `myfunc` module has been added to the Solverz path, the numerical implementations can be called by +`myfunc.Min`. + +After finish the above procedures, we can finally use the `Min` function in our simulation modelling, with, for example, + ```python -@njit(cache=True) -def SolLessThan(x, y): - x = np.asarray(x).reshape((-1,)) - return (x < y).astype(np.int32) +from Solverz import Model, Var, Eqn, made_numerical, MulVarFunc +from Min import Min + +m = Model() +m.x = Var('x', [1, 2]) +m.y = Var('y', [3, 4]) +m.f = Eqn('f', Min(m.x, m.y)) +sae, y0 = m.create_instance() +ae = made_numerical(sae, y0, sparse=True) ``` -The `implements_nfunc()` cannot be omitted and the `njit()` decorator enables the numba-based dynamic compilation for efficiency. +We will have the output + +```shell +>>> ae.F(y0) +array([1.0, 2.0]) +``` From f12e3fe4a58637b25468dc5a6e0809219962976d Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Fri, 22 Nov 2024 14:54:15 +0800 Subject: [PATCH 08/12] refactor: implement UDM using editable package --- Solverz/__init__.py | 1 - Solverz/num_api/module_parser.py | 17 ++-- Solverz/num_api/test/test_udm.py | 90 +++++++------------ Solverz/num_api/user_function_parser.py | 111 ------------------------ docs/src/advanced.md | 2 +- 5 files changed, 37 insertions(+), 184 deletions(-) delete mode 100644 Solverz/num_api/user_function_parser.py diff --git a/Solverz/__init__.py b/Solverz/__init__.py index 1f43a57..fe26925 100644 --- a/Solverz/__init__.py +++ b/Solverz/__init__.py @@ -11,7 +11,6 @@ from Solverz.utilities.profile import count_time from Solverz.variable.ssymbol import Var, AliasVar from Solverz.model.basic import Model -from Solverz.num_api.user_function_parser import add_my_module, reset_my_module_paths from importlib.metadata import version, PackageNotFoundError diff --git a/Solverz/num_api/module_parser.py b/Solverz/num_api/module_parser.py index 86709d6..513f3d0 100644 --- a/Solverz/num_api/module_parser.py +++ b/Solverz/num_api/module_parser.py @@ -18,20 +18,13 @@ warnings.warn(f'Failed to import num api from SolMuseum: {e}') # parse user defined functions -from .user_function_parser import load_my_module_paths -user_module_paths = load_my_module_paths() -if user_module_paths: +try: + import myfunc print('User module detected.') - import os, sys - for path in user_module_paths: - module_name = os.path.splitext(os.path.basename(path))[0] - module_dir = os.path.dirname(path) - - sys.path.insert(0, module_dir) - exec('import ' + module_name) - module_dict[module_name] = globals()[module_name] - + module_dict['myfunc'] = myfunc +except ModuleNotFoundError as e: + pass modules = [module_dict, 'numpy'] # We preserve the 'numpy' here in case one uses functions from sympy instead of from Solverz diff --git a/Solverz/num_api/test/test_udm.py b/Solverz/num_api/test/test_udm.py index b713401..3cb00a7 100644 --- a/Solverz/num_api/test/test_udm.py +++ b/Solverz/num_api/test/test_udm.py @@ -2,74 +2,46 @@ Test the user defined modules. """ -import importlib -import os -import re -import shutil -from pathlib import Path - -import pytest - -from Solverz.num_api.user_function_parser import add_my_module, reset_my_module_paths - -mymodule_code = """import numpy as np -from numba import njit - - -@njit(cache=True) -def c(x, y): - x = np.asarray(x).reshape((-1,)) - y = np.asarray(y).reshape((-1,)) - - z = np.zeros_like(x) - - for i in range(len(x)): - if x[i] <= y[i]: - z[i] = x[i] - else: - z[i] = y[i] - return z -""" - def test_udm(): - # Create a .Solverz_test_temp directory in the user's home directory - user_home = str(Path.home()) - solverz_dir = os.path.join(user_home, '.Solverz_test_temp') - # Create the .Solverz directory if it does not exist - if not os.path.exists(solverz_dir): - os.makedirs(solverz_dir) + from Solverz import Model, Var, Eqn, made_numerical, MulVarFunc + import numpy as np - file_path = os.path.join(solverz_dir, r'your_module.py') - file_path1 = os.path.join(solverz_dir, r'fake1.jl') + class Min(MulVarFunc): + arglength = 2 - # Write the new paths to the file, but only if they are not already present - with open(file_path, 'a') as file: - file.write(mymodule_code) + def fdiff(self, argindex=1): + if argindex == 1: + return dMindx(*self.args) + elif argindex == 2: + return dMindy(*self.args) - with open(file_path1, 'a') as file: - file.write(mymodule_code) + def _numpycode(self, printer, **kwargs): + return (f'myfunc.Min' + r'(' + + ', '.join([printer._print(arg, **kwargs) for arg in self.args]) + r')') - with pytest.raises(ValueError, - match=re.escape(f"The path {solverz_dir} is not a file.")): - add_my_module([solverz_dir]) + class dMindx(MulVarFunc): + arglength = 2 - with pytest.raises(ValueError, - match=re.escape(f"The path {os.path.join(user_home, '.Solverz_test_temp1')} does not exist.")): - add_my_module([os.path.join(user_home, '.Solverz_test_temp1')]) + def _numpycode(self, printer, **kwargs): + return (f'myfunc.dMindx' + r'(' + + ', '.join([printer._print(arg, **kwargs) for arg in self.args]) + r')') - with pytest.raises(ValueError, - match=re.escape(f"The file {file_path1} is not a Python file.")): - add_my_module([file_path1]) + class dMindy(MulVarFunc): + arglength = 2 - add_my_module([file_path]) + def _numpycode(self, printer, **kwargs): + return (f'myfunc.dMindy' + r'(' + + ', '.join([printer._print(arg, **kwargs) for arg in self.args]) + r')') - import Solverz - importlib.reload(Solverz.num_api.module_parser) - from Solverz.num_api.module_parser import your_module - import numpy as np - np.testing.assert_allclose(your_module.c(np.array([1, 0]), np.array([2, -1])), np.array([1, -1])) + m = Model() + m.x = Var('x', [1, 2]) + m.y = Var('y', [3, 4]) + m.f = Eqn('f', Min(m.x, m.y)) + sae, y0 = m.create_instance() + ae = made_numerical(sae, y0, sparse=True) + np.testing.assert_allclose(ae.F(y0, ae.p), np.array([1.0, 2.0])) + np.testing.assert_allclose(ae.J(y0, ae.p).toarray(), np.array([[1., 0., 0., 0.], + [0., 1., 0., 0.]])) - shutil.rmtree(solverz_dir) - reset_my_module_paths() diff --git a/Solverz/num_api/user_function_parser.py b/Solverz/num_api/user_function_parser.py deleted file mode 100644 index f10c5df..0000000 --- a/Solverz/num_api/user_function_parser.py +++ /dev/null @@ -1,111 +0,0 @@ -import os -from pathlib import Path - - -def validate_module_paths(paths): - """ - Validate that each path in the provided list points to a valid Python module. - - :param paths: List of paths - :return: List of validated paths - :raises ValueError: If a path is invalid or not a Python module - """ - valid_paths = [] - for path in paths: - # Check if the path exists - if not os.path.exists(path): - raise ValueError(f"The path {path} does not exist.") - - # Check if the path is a file - if not os.path.isfile(path): - raise ValueError(f"The path {path} is not a file.") - - # Check if the file is a Python file - if not path.endswith('.py'): - raise ValueError(f"The file {path} is not a Python file.") - - # If all checks pass, add the path to the valid paths list - valid_paths.append(path) - - return valid_paths - - -def add_my_module(paths, filename='user_modules.txt'): - """ - Save user-provided module paths to a specified file, but validate the paths before saving. - If a path already exists in the file, it will not be added again. - - :param paths: List of user-provided module paths - :param filename: Name of the file to save, default is 'user_modules.txt' - """ - - # Validate paths - validated_paths = validate_module_paths(paths) - - # Get the path to the .Solverz directory in the user's home directory - user_home = str(Path.home()) - solverz_dir = os.path.join(user_home, '.Solverz') - - # Create the .Solverz directory if it does not exist - if not os.path.exists(solverz_dir): - os.makedirs(solverz_dir) - - # Define the full path to the file - file_path = os.path.join(solverz_dir, filename) - - # Read existing paths from the file - existing_paths = set() - if os.path.exists(file_path): - with open(file_path, 'r') as file: - existing_paths = set(line.strip() for line in file) - - # Write the new paths to the file, but only if they are not already present - with open(file_path, 'a') as file: - for path in validated_paths: - if path not in existing_paths: - file.write(f'{path}\n') - existing_paths.add(path) - - -def load_my_module_paths(filename='user_modules.txt'): - """ - Load module paths from a specified file in the .Solverz directory in the user's home directory. - - :param filename: Name of the file to load, default is 'user_modules.txt' - :return: List of module paths - """ - user_home = str(Path.home()) - solverz_dir = os.path.join(user_home, '.Solverz') - file_path = os.path.join(solverz_dir, filename) - - # Check if the file exists - if not os.path.exists(file_path): - return [] - - # Read and return the list of paths - with open(file_path, 'r') as file: - paths = [line.strip() for line in file] - - return paths - - -def reset_my_module_paths(filename='user_modules.txt'): - """ - Reset the user_modules.txt file by clearing its content. - - :param filename: Name of the file to reset, default is 'user_modules.txt' - """ - # Get the path to the .Solverz directory in the user's home directory - user_home = str(Path.home()) - solverz_dir = os.path.join(user_home, '.Solverz') - - # Define the full path to the file - file_path = os.path.join(solverz_dir, filename) - - # Create the .Solverz directory if it does not exist - if not os.path.exists(solverz_dir): - os.makedirs(solverz_dir) - - # Clear the content of the file - with open(file_path, 'w') as file: - file.write('') diff --git a/docs/src/advanced.md b/docs/src/advanced.md index ee629bd..f994134 100644 --- a/docs/src/advanced.md +++ b/docs/src/advanced.md @@ -187,6 +187,6 @@ ae = made_numerical(sae, y0, sparse=True) We will have the output ```shell ->>> ae.F(y0) +>>> ae.F(y0, ae.p) array([1.0, 2.0]) ``` From 57c887d26242f89f50b45e5425897b4f9de7570a Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Fri, 22 Nov 2024 15:05:59 +0800 Subject: [PATCH 09/12] test: install `myfunc` first --- .github/workflows/run-tests.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index cbec3c3..be3dd96 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -13,8 +13,20 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - name: Checkout myfunc Repository + uses: actions/checkout@v3 + with: + repository: 'smallbunnies/myfunc' + ref: 'main' - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - run: | + pip install -e . + + + - uname: Return to original repo + uses: actions/checkout@v3 with: python-version: '3.11' cache: 'pip' # caching pip dependencies From 41a859638df1554cac02ee93462ee3e13d8d9d89 Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Fri, 22 Nov 2024 15:07:07 +0800 Subject: [PATCH 10/12] Update run-tests.yml --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index be3dd96..6202053 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -25,7 +25,7 @@ jobs: pip install -e . - - uname: Return to original repo + - name: Return to original repo uses: actions/checkout@v3 with: python-version: '3.11' From 30e746bfe0e510f54c435678e12e2c1c373a71c0 Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Fri, 22 Nov 2024 15:15:21 +0800 Subject: [PATCH 11/12] Update run-tests.yml --- .github/workflows/run-tests.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6202053..97c9c53 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -13,24 +13,13 @@ jobs: runs-on: windows-latest steps: - - name: Checkout myfunc Repository - uses: actions/checkout@v3 - with: - repository: 'smallbunnies/myfunc' - ref: 'main' + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - run: | - pip install -e . - - - - name: Return to original repo - uses: actions/checkout@v3 with: python-version: '3.11' cache: 'pip' # caching pip dependencies - run: | pip install -e . + pip install git+https://github.com/smallbunnies/myfunc.git@main - run: | # run both independent pytest and doctest pytest From 9a43302b983f57ff4770613c2aae02d74df3d91b Mon Sep 17 00:00:00 2001 From: Ruizhi Yu Date: Fri, 22 Nov 2024 15:47:15 +0800 Subject: [PATCH 12/12] docs: update UDM usage --- docs/src/advanced.md | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/src/advanced.md b/docs/src/advanced.md index f994134..5d795d6 100644 --- a/docs/src/advanced.md +++ b/docs/src/advanced.md @@ -58,7 +58,7 @@ First, we define the numerical interfaces. The derivatives of $\min$ function ar \end{cases}. ``` -Let us create a `myfunc.py` file and put the numerical codes there, which will look like +Let us create a `myfunc` directory and put the numerical codes in the `myfunc.py` file that looks like ```python # myfunc.py @@ -101,18 +101,26 @@ def dMindy(x, y): return 1-dMindx(x, y) ``` -where we use `Min` to avoid conflicting with the built-in `min` function, -the `@njit(cache)` decorator is used to perform the jit-compilation and hence speed up the numerical codes. +In `myfunc.py`, we use `Min` to avoid conflicting with the built-in `min` function. +The `@njit(cache)` decorator is used to perform the jit-compilation and hence speed up the numerical codes. -Then let us add the path of `myfunc.py` to Solverz, with, for example, +Then let us install the `myfunc` module, so that Solverz can import the `myfunc` module. Use the terminal to switch +to the `myfunc` module directory. Add a `pyproject.toml` file there. -```python -from Solverz import add_my_module -add_my_module([r'D:\myfunc.py']) +```{note} +One can clone the `pyproject.toml` file from [example repo](https://github.com/smallbunnies/myfunc). +``` + +Use the following command to install the module. + +```shell +pip install -e . ``` -Then Solverz would load your functions defined in the `myfunc` module. One can also reset the paths by calling -`reset_my_module_paths`. +```{note} +Because `myfunc` module is installed in the editable mode, one can change the numerical implementations in `myfunc.py` +with great freedom. +``` As for the symbolic implementation, let us start by creating a `Min.py` file and subclassing `MulVarFunc` there with @@ -167,13 +175,14 @@ class dMindy(MulVarFunc): where the `fdiff` function should return the derivative of the function, without considering the chain rule, with respect to the `argindex`-th variable; the `_numpycode` functions define the numerical implementations of the -functions. Since `myfunc` module has been added to the Solverz path, the numerical implementations can be called by +functions. Since the `myfunc` module has been installed, the numerical implementations can be called by `myfunc.Min`. -After finish the above procedures, we can finally use the `Min` function in our simulation modelling, with, for example, +After finish the above procedures, we can finally use the `Min` function in our simulation modelling. An example is as +follows. ```python -from Solverz import Model, Var, Eqn, made_numerical, MulVarFunc +from Solverz import Model, Var, Eqn, made_numerical from Min import Min m = Model()