From 7724e55e2ac75a3817b66851035dea2378729818 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 14 Mar 2024 13:44:04 -0500 Subject: [PATCH] feat: Package matplotlib and ui JS with wheel (#343) Fixes #194, fixes #316 Packages matplotlib and ui JS with wheel using new util packages. I also took the opportunity to refactor those two a bit to match the plotly express package by moving registration out of the `__init__.py` files. Also refactored plotly express. I've verified the installed wheels work properly with basic examples. --------- Co-authored-by: Mike Bender --- .pre-commit-config.yaml | 3 +- cog.toml | 2 + docker/config/deephaven.prop | 3 -- plugins/matplotlib/pyproject.toml | 2 +- plugins/matplotlib/setup.cfg | 6 +-- plugins/matplotlib/setup.py | 16 ++++++++ .../deephaven/plugin/matplotlib/__init__.py | 23 ----------- .../deephaven/plugin/matplotlib/_js_plugin.py | 32 +++++++++++++++ .../deephaven/plugin/matplotlib/_register.py | 39 ++++++++++++++++++ plugins/plotly-express/pyproject.toml | 2 +- plugins/plotly-express/setup.cfg | 1 + plugins/plotly-express/setup.py | 32 +-------------- .../src/deephaven/plot/express/_js_plugin.py | 41 ------------------- .../src/deephaven/plot/express/_register.py | 21 +++++++--- plugins/ui/pyproject.toml | 2 +- plugins/ui/setup.cfg | 5 ++- plugins/ui/setup.py | 10 +++++ plugins/ui/src/deephaven/ui/__init__.py | 10 ----- plugins/ui/src/deephaven/ui/_js_plugin.py | 32 +++++++++++++++ plugins/ui/src/deephaven/ui/_register.py | 27 ++++++++++++ tools/update_version.sh | 2 +- 21 files changed, 187 insertions(+), 124 deletions(-) create mode 100644 plugins/matplotlib/setup.py create mode 100644 plugins/matplotlib/src/deephaven/plugin/matplotlib/_js_plugin.py create mode 100644 plugins/matplotlib/src/deephaven/plugin/matplotlib/_register.py create mode 100644 plugins/ui/setup.py create mode 100644 plugins/ui/src/deephaven/ui/_js_plugin.py create mode 100644 plugins/ui/src/deephaven/ui/_register.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ef4506e65..69809928c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,8 @@ repos: deephaven-core, plotly, json-rpc, - matplotlib + matplotlib, + deephaven-plugin-utilities ] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.2.2 diff --git a/cog.toml b/cog.toml index d84f61c28..6d8c99871 100644 --- a/cog.toml +++ b/cog.toml @@ -76,4 +76,6 @@ plotly = { path = "plugins/plotly", public_api=false } plotly-express = { path = "plugins/plotly-express", public_api=false } table-example = { path = "plugins/table-example", public_api=false } ui = { path = "plugins/ui", public_api=false } +packaging = { path = "plugins/packaging", public_api=false } +utilities = { path = "plugins/utilities", public_api=false } diff --git a/docker/config/deephaven.prop b/docker/config/deephaven.prop index 114ab0959..95662a24c 100644 --- a/docker/config/deephaven.prop +++ b/docker/config/deephaven.prop @@ -2,9 +2,6 @@ includefiles=dh-defaults.prop deephaven.console.type=python -# Add all plugins that you want installed here -deephaven.jsPlugins.@deephaven/js-plugin-matplotlib=/opt/deephaven/config/plugins/plugins/matplotlib/src/js -deephaven.jsPlugins.@deephaven/js-plugin-ui=/opt/deephaven/config/plugins/plugins/ui/src/js # Anonymous authentication so we don't need to put in a password AuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler \ No newline at end of file diff --git a/plugins/matplotlib/pyproject.toml b/plugins/matplotlib/pyproject.toml index 62df2b006..54e217464 100644 --- a/plugins/matplotlib/pyproject.toml +++ b/plugins/matplotlib/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools>=43.0.0", "wheel"] +requires = ["setuptools>=43.0.0", "wheel", "deephaven-plugin-packaging"] build-backend = "setuptools.build_meta" diff --git a/plugins/matplotlib/setup.cfg b/plugins/matplotlib/setup.cfg index 5052d68f7..2f1f71a99 100644 --- a/plugins/matplotlib/setup.cfg +++ b/plugins/matplotlib/setup.cfg @@ -29,6 +29,7 @@ install_requires = jpy>=0.14.0 deephaven-plugin>=0.5.0 matplotlib + deephaven-plugin-utilities include_package_data = True [options.extras_require] @@ -38,9 +39,6 @@ seaborn = [options.packages.find] where=src -[options.package_data] -* = *.mplstyle - [options.entry_points] deephaven.plugin = - registration_cls = deephaven.plugin.matplotlib:MatplotlibRegistration \ No newline at end of file + registration_cls = deephaven.plugin.matplotlib._register:MatplotlibRegistration \ No newline at end of file diff --git a/plugins/matplotlib/setup.py b/plugins/matplotlib/setup.py new file mode 100644 index 000000000..b8d654031 --- /dev/null +++ b/plugins/matplotlib/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup +import os + +from deephaven.plugin.packaging import package_js + +js_dir = "src/js/" +dest_dir = os.path.join("src/deephaven/plugin/matplotlib/_js") + +package_js(js_dir, dest_dir) + +setup( + package_data={ + "deephaven.plugin.matplotlib._js": ["**"], + "deephaven.plugin.matplotlib": ["deephaven.mplstyle"], + } +) diff --git a/plugins/matplotlib/src/deephaven/plugin/matplotlib/__init__.py b/plugins/matplotlib/src/deephaven/plugin/matplotlib/__init__.py index 17491fe08..590530d9e 100644 --- a/plugins/matplotlib/src/deephaven/plugin/matplotlib/__init__.py +++ b/plugins/matplotlib/src/deephaven/plugin/matplotlib/__init__.py @@ -1,32 +1,9 @@ from deephaven import numpy as dhnp -from deephaven.plugin import Registration, Callback from deephaven.table_listener import listen -from importlib import resources -import matplotlib.pyplot as plt from matplotlib.animation import Animation import itertools -def _init_theme(): - # Set the Deephaven style globally. - # We use the savefig function to export the Figure, and that uses the Figure's properties for colours rather than temporary styling. - # The Figure's properties are set on creation time of the Figure, rather than when the Figure is exported - # We do not have hooks into when a user creates a new Figure, so we set the theme globally ahead of time - # https://github.com/matplotlib/matplotlib/issues/6592/ - with resources.path(__package__, "deephaven.mplstyle") as p: - plt.style.use(["dark_background", p]) - - -class MatplotlibRegistration(Registration): - @classmethod - def register_into(cls, callback: Callback) -> None: - _init_theme() - plt.switch_backend("AGG") - from . import figure_type - - callback.register(figure_type.FigureType) - - class TableEventSource: """ Makes an event source for matplotlib that triggers whenever Deephaven Table updates. diff --git a/plugins/matplotlib/src/deephaven/plugin/matplotlib/_js_plugin.py b/plugins/matplotlib/src/deephaven/plugin/matplotlib/_js_plugin.py new file mode 100644 index 000000000..7c2e9bf65 --- /dev/null +++ b/plugins/matplotlib/src/deephaven/plugin/matplotlib/_js_plugin.py @@ -0,0 +1,32 @@ +import pathlib + +from deephaven.plugin.js import JsPlugin + + +class MatplotlibJsPlugin(JsPlugin): + def __init__( + self, + name: str, + version: str, + main: str, + path: pathlib.Path, + ) -> None: + self._name = name + self._version = version + self._main = main + self._path = path + + @property + def name(self) -> str: + return self._name + + @property + def version(self) -> str: + return self._version + + @property + def main(self) -> str: + return self._main + + def path(self) -> pathlib.Path: + return self._path diff --git a/plugins/matplotlib/src/deephaven/plugin/matplotlib/_register.py b/plugins/matplotlib/src/deephaven/plugin/matplotlib/_register.py new file mode 100644 index 000000000..33d18567a --- /dev/null +++ b/plugins/matplotlib/src/deephaven/plugin/matplotlib/_register.py @@ -0,0 +1,39 @@ +from importlib import resources +import matplotlib.pyplot as plt +from deephaven.plugin import Registration, Callback +from deephaven.plugin.utilities import create_js_plugin, DheSafeCallbackWrapper +from ._js_plugin import MatplotlibJsPlugin + +PACKAGE_NAMESPACE = "deephaven.plugin.matplotlib" +JS_NAME = "_js" +PLUGIN_CLASS = MatplotlibJsPlugin + + +def _init_theme(): + # Set the Deephaven style globally. + # We use the savefig function to export the Figure, and that uses the Figure's properties for colours rather than temporary styling. + # The Figure's properties are set on creation time of the Figure, rather than when the Figure is exported + # We do not have hooks into when a user creates a new Figure, so we set the theme globally ahead of time + # https://github.com/matplotlib/matplotlib/issues/6592/ + with resources.path(__package__, "deephaven.mplstyle") as p: + plt.style.use(["dark_background", p]) + + +class MatplotlibRegistration(Registration): + @classmethod + def register_into(cls, callback: Callback) -> None: + _init_theme() + plt.switch_backend("AGG") + from . import figure_type + + callback = DheSafeCallbackWrapper(callback) + + callback.register(figure_type.FigureType) + + js_plugin = create_js_plugin( + PACKAGE_NAMESPACE, + JS_NAME, + PLUGIN_CLASS, + ) + + callback.register(js_plugin) diff --git a/plugins/plotly-express/pyproject.toml b/plugins/plotly-express/pyproject.toml index 62df2b006..54e217464 100644 --- a/plugins/plotly-express/pyproject.toml +++ b/plugins/plotly-express/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools>=43.0.0", "wheel"] +requires = ["setuptools>=43.0.0", "wheel", "deephaven-plugin-packaging"] build-backend = "setuptools.build_meta" diff --git a/plugins/plotly-express/setup.cfg b/plugins/plotly-express/setup.cfg index 9b213314b..501456df6 100644 --- a/plugins/plotly-express/setup.cfg +++ b/plugins/plotly-express/setup.cfg @@ -27,6 +27,7 @@ packages=find_namespace: install_requires = deephaven-plugin>=0.6.0 plotly + deephaven-plugin-utilities include_package_data = True [options.packages.find] diff --git a/plugins/plotly-express/setup.py b/plugins/plotly-express/setup.py index 730690ce9..484dcc8aa 100644 --- a/plugins/plotly-express/setup.py +++ b/plugins/plotly-express/setup.py @@ -1,38 +1,10 @@ -import shutil from setuptools import setup import os -import subprocess - -# Uses npm pack to create a tarball of the package and unpacks it into a build directory -# Then uses that to add to the wheel +from deephaven.plugin.packaging import package_js js_dir = "src/js/" -dist_dir = os.path.join(js_dir, "dist") -build_dir = os.path.join(js_dir, "build") dest_dir = os.path.join("src/deephaven/plot/express/_js") -package_dir = os.path.join(build_dir, "package") - -# copy the bundle to the plotly-express directory -# the path may not exist (e.g. when running tests) -# so it is not strictly necessary to copy the bundle -if os.path.exists(dist_dir): - # ignore errors as the directory may not exist - shutil.rmtree(build_dir, ignore_errors=True) - shutil.rmtree(dest_dir, ignore_errors=True) - - os.makedirs(build_dir, exist_ok=True) - - # pack and unpack into the js plotly-express directory - subprocess.run( - ["npm", "pack", "--pack-destination", "build"], cwd=js_dir, check=True - ) - # it is assumed that there is only one tarball in the directory - files = os.listdir(build_dir) - for file in files: - subprocess.run(["tar", "-xzf", file], cwd=build_dir, check=True) - os.remove(os.path.join(build_dir, file)) - # move the package directory to the expected package location - shutil.move(package_dir, dest_dir) +package_js(js_dir, dest_dir) setup(package_data={"deephaven.plot.express._js": ["**"]}) diff --git a/plugins/plotly-express/src/deephaven/plot/express/_js_plugin.py b/plugins/plotly-express/src/deephaven/plot/express/_js_plugin.py index 3df3011f9..f1df67cba 100644 --- a/plugins/plotly-express/src/deephaven/plot/express/_js_plugin.py +++ b/plugins/plotly-express/src/deephaven/plot/express/_js_plugin.py @@ -1,8 +1,4 @@ -import importlib.resources -import json import pathlib -import sys -from typing import Callable, ContextManager from deephaven.plugin.js import JsPlugin @@ -34,40 +30,3 @@ def main(self) -> str: def path(self) -> pathlib.Path: return self._path - - -def _create_from_npm_package_json( - path_provider: Callable[[], ContextManager[pathlib.Path]] -) -> JsPlugin: - with path_provider() as tmp_js_path: - js_path = tmp_js_path - if not js_path.exists(): - raise Exception( - f"Package is not installed in a normal python filesystem, '{js_path}' does not exist" - ) - with (js_path / "package.json").open("rb") as f: - package_json = json.load(f) - return ExpressJsPlugin( - package_json["name"], - package_json["version"], - package_json["main"], - js_path, - ) - - -def _resource_js_path() -> ContextManager[pathlib.Path]: - namespace = "deephaven.plot.express" - name = "_js" - if sys.version_info < (3, 9): - return importlib.resources.path(namespace, name) - else: - return importlib.resources.as_file( - importlib.resources.files(namespace).joinpath(name) - ) - - -def create_js_plugin() -> JsPlugin: - # TODO: Include developer instructions for installing in editable mode - # https://github.com/deephaven/deephaven-plugins/issues/93 - # TBD what editable mode looks like for JsPlugin - return _create_from_npm_package_json(_resource_js_path) diff --git a/plugins/plotly-express/src/deephaven/plot/express/_register.py b/plugins/plotly-express/src/deephaven/plot/express/_register.py index f3f638ba5..34d2383ee 100644 --- a/plugins/plotly-express/src/deephaven/plot/express/_register.py +++ b/plugins/plotly-express/src/deephaven/plot/express/_register.py @@ -1,8 +1,11 @@ -import os +from deephaven.plugin import Registration, Callback +from deephaven.plugin.utilities import create_js_plugin, DheSafeCallbackWrapper from . import DeephavenFigureType -from ._js_plugin import create_js_plugin +from ._js_plugin import ExpressJsPlugin -from deephaven.plugin import Registration, Callback +PACKAGE_NAMESPACE = "deephaven.plot.express" +JS_NAME = "_js" +PLUGIN_CLASS = ExpressJsPlugin class ExpressRegistration(Registration): @@ -21,8 +24,14 @@ def register_into(cls, callback: Callback) -> None: A function to call after registration """ + callback = DheSafeCallbackWrapper(callback) + callback.register(DeephavenFigureType) - # Only register the JS plugins if the environment variable is set - if os.getenv("DEEPHAVEN_ENABLE_PY_JS", "False").lower() in ("true", "1"): - callback.register(create_js_plugin()) + js_plugin = create_js_plugin( + PACKAGE_NAMESPACE, + JS_NAME, + PLUGIN_CLASS, + ) + + callback.register(js_plugin) diff --git a/plugins/ui/pyproject.toml b/plugins/ui/pyproject.toml index 62df2b006..54e217464 100644 --- a/plugins/ui/pyproject.toml +++ b/plugins/ui/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools>=43.0.0", "wheel"] +requires = ["setuptools>=43.0.0", "wheel", "deephaven-plugin-packaging"] build-backend = "setuptools.build_meta" diff --git a/plugins/ui/setup.cfg b/plugins/ui/setup.cfg index 14e5882a4..7ecfaa886 100644 --- a/plugins/ui/setup.cfg +++ b/plugins/ui/setup.cfg @@ -3,7 +3,7 @@ name = deephaven-plugin-ui description = deephaven.ui plugin long_description = file: README.md long_description_content_type = text/markdown -version = attr:deephaven.ui.__version__ +version = 0.9.0.dev0 url = https://github.com/deephaven/deephaven-plugins project_urls = Source Code = https://github.com/deephaven/deephaven-plugins @@ -28,6 +28,7 @@ install_requires = deephaven-core>=0.31.0 deephaven-plugin>=0.6.0 json-rpc + deephaven-plugin-utilities typing_extensions include_package_data = True @@ -36,4 +37,4 @@ where=src [options.entry_points] deephaven.plugin = - registration_cls = deephaven.ui:UIRegistration + registration_cls = deephaven.ui._register:UIRegistration diff --git a/plugins/ui/setup.py b/plugins/ui/setup.py new file mode 100644 index 000000000..29b112d5b --- /dev/null +++ b/plugins/ui/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup +import os +from deephaven.plugin.packaging import package_js + +js_dir = "src/js/" +dest_dir = os.path.join("src/deephaven/ui/_js") + +package_js(js_dir, dest_dir) + +setup(package_data={"deephaven.ui._js": ["**"]}) diff --git a/plugins/ui/src/deephaven/ui/__init__.py b/plugins/ui/src/deephaven/ui/__init__.py index fad0cf4d9..3b5ab4f7a 100644 --- a/plugins/ui/src/deephaven/ui/__init__.py +++ b/plugins/ui/src/deephaven/ui/__init__.py @@ -4,17 +4,7 @@ The API is designed to be similar to React, but with some differences to make it more Pythonic. """ -from deephaven.plugin import Registration, Callback from .components import * from .elements import * from .hooks import * from .object_types import * - -__version__ = "0.9.0.dev0" - - -class UIRegistration(Registration): - @classmethod - def register_into(cls, callback: Callback) -> None: - callback.register(DashboardType) - callback.register(ElementType) diff --git a/plugins/ui/src/deephaven/ui/_js_plugin.py b/plugins/ui/src/deephaven/ui/_js_plugin.py new file mode 100644 index 000000000..fa3b9a05e --- /dev/null +++ b/plugins/ui/src/deephaven/ui/_js_plugin.py @@ -0,0 +1,32 @@ +import pathlib + +from deephaven.plugin.js import JsPlugin + + +class UiJsPlugin(JsPlugin): + def __init__( + self, + name: str, + version: str, + main: str, + path: pathlib.Path, + ) -> None: + self._name = name + self._version = version + self._main = main + self._path = path + + @property + def name(self) -> str: + return self._name + + @property + def version(self) -> str: + return self._version + + @property + def main(self) -> str: + return self._main + + def path(self) -> pathlib.Path: + return self._path diff --git a/plugins/ui/src/deephaven/ui/_register.py b/plugins/ui/src/deephaven/ui/_register.py new file mode 100644 index 000000000..3dfe4502b --- /dev/null +++ b/plugins/ui/src/deephaven/ui/_register.py @@ -0,0 +1,27 @@ +from . import DashboardType, ElementType +from deephaven.plugin import Registration, Callback +from deephaven.plugin.utilities import create_js_plugin, DheSafeCallbackWrapper + +from ._js_plugin import UiJsPlugin + +PACKAGE_NAMESPACE = "deephaven.ui" +JS_NAME = "_js" +PLUGIN_CLASS = UiJsPlugin + + +class UIRegistration(Registration): + @classmethod + def register_into(cls, callback: Callback) -> None: + + callback = DheSafeCallbackWrapper(callback) + + callback.register(DashboardType) + callback.register(ElementType) + + js_plugin = create_js_plugin( + PACKAGE_NAMESPACE, + JS_NAME, + PLUGIN_CLASS, + ) + + callback.register(js_plugin) diff --git a/tools/update_version.sh b/tools/update_version.sh index 0eee3ab85..4f89bdf24 100755 --- a/tools/update_version.sh +++ b/tools/update_version.sh @@ -137,7 +137,7 @@ case "$package" in ;; ui) update_file ui/src/js/package.json '"version": "' '",' - update_file ui/src/deephaven/ui/__init__.py '__version__ = "' '"' "$extra" + update_file ui/src/setup.cfg 'version = ' '' "$extra" ;; utilities) update_file utilities/setup.cfg 'version = ' '' "$extra"