feat: Package matplotlib and ui JS with wheel (#343)

Fixes #194, fixes #316 

Packages matplotlib and ui JS with wheel using new util packages. I also
took the opportunity to refactor those two a bit to match the plotly
express package by moving registration out of the `__init__.py` files.
Also refactored plotly express.

I've verified the installed wheels work properly with basic examples.


---------

Co-authored-by: Mike Bender <[email protected]>

.pre-commit-config.yaml

@@ -20,7 +20,8 @@ repos:
             deephaven-core,
             plotly,
             json-rpc,
-            matplotlib
+            matplotlib,
+            deephaven-plugin-utilities
           ]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.2.2

cog.toml

@@ -76,4 +76,6 @@ plotly = { path = "plugins/plotly", public_api=false }
 plotly-express = { path = "plugins/plotly-express", public_api=false }
 table-example = { path = "plugins/table-example", public_api=false }
 ui = { path = "plugins/ui", public_api=false }
+packaging = { path = "plugins/packaging", public_api=false }
+utilities = { path = "plugins/utilities", public_api=false }
 

docker/config/deephaven.prop

@@ -2,9 +2,6 @@ includefiles=dh-defaults.prop
 
 deephaven.console.type=python
 
-# Add all plugins that you want installed here
-deephaven.jsPlugins.@deephaven/js-plugin-matplotlib=/opt/deephaven/config/plugins/plugins/matplotlib/src/js
-deephaven.jsPlugins.@deephaven/js-plugin-ui=/opt/deephaven/config/plugins/plugins/ui/src/js
 
 # Anonymous authentication so we don't need to put in a password
 AuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler

plugins/matplotlib/pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools>=43.0.0", "wheel"]
+requires = ["setuptools>=43.0.0", "wheel", "deephaven-plugin-packaging"]
 build-backend = "setuptools.build_meta"

plugins/matplotlib/setup.cfg

@@ -29,6 +29,7 @@ install_requires =
     jpy>=0.14.0
     deephaven-plugin>=0.5.0
     matplotlib
+    deephaven-plugin-utilities
 include_package_data = True
 
 [options.extras_require]
@@ -38,9 +39,6 @@ seaborn =
 [options.packages.find]
 where=src
 
-[options.package_data]
-* = *.mplstyle
-
 [options.entry_points]
 deephaven.plugin =
-    registration_cls = deephaven.plugin.matplotlib:MatplotlibRegistration
+    registration_cls = deephaven.plugin.matplotlib._register:MatplotlibRegistration

plugins/matplotlib/setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup
+import os
+
+from deephaven.plugin.packaging import package_js
+
+js_dir = "src/js/"
+dest_dir = os.path.join("src/deephaven/plugin/matplotlib/_js")
+
+package_js(js_dir, dest_dir)
+
+setup(
+    package_data={
+        "deephaven.plugin.matplotlib._js": ["**"],
+        "deephaven.plugin.matplotlib": ["deephaven.mplstyle"],
+    }
+)

plugins/matplotlib/src/deephaven/plugin/matplotlib/__init__.py

@@ -1,32 +1,9 @@
 from deephaven import numpy as dhnp
-from deephaven.plugin import Registration, Callback
 from deephaven.table_listener import listen
-from importlib import resources
-import matplotlib.pyplot as plt
 from matplotlib.animation import Animation
 import itertools
 
 
-def _init_theme():
-    # Set the Deephaven style globally.
-    # We use the savefig function to export the Figure, and that uses the Figure's properties for colours rather than temporary styling.
-    # The Figure's properties are set on creation time of the Figure, rather than when the Figure is exported
-    # We do not have hooks into when a user creates a new Figure, so we set the theme globally ahead of time
-    # https://github.com/matplotlib/matplotlib/issues/6592/
-    with resources.path(__package__, "deephaven.mplstyle") as p:
-        plt.style.use(["dark_background", p])
-
-
-class MatplotlibRegistration(Registration):
-    @classmethod
-    def register_into(cls, callback: Callback) -> None:
-        _init_theme()
-        plt.switch_backend("AGG")
-        from . import figure_type
-
-        callback.register(figure_type.FigureType)
-
-
 class TableEventSource:
     """
     Makes an event source for matplotlib that triggers whenever Deephaven Table updates.

plugins/matplotlib/src/deephaven/plugin/matplotlib/_js_plugin.py

@@ -0,0 +1,32 @@
+import pathlib
+
+from deephaven.plugin.js import JsPlugin
+
+
+class MatplotlibJsPlugin(JsPlugin):
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        main: str,
+        path: pathlib.Path,
+    ) -> None:
+        self._name = name
+        self._version = version
+        self._main = main
+        self._path = path
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def version(self) -> str:
+        return self._version
+
+    @property
+    def main(self) -> str:
+        return self._main
+
+    def path(self) -> pathlib.Path:
+        return self._path

plugins/matplotlib/src/deephaven/plugin/matplotlib/_register.py

@@ -0,0 +1,39 @@
+from importlib import resources
+import matplotlib.pyplot as plt
+from deephaven.plugin import Registration, Callback
+from deephaven.plugin.utilities import create_js_plugin, DheSafeCallbackWrapper
+from ._js_plugin import MatplotlibJsPlugin
+
+PACKAGE_NAMESPACE = "deephaven.plugin.matplotlib"
+JS_NAME = "_js"
+PLUGIN_CLASS = MatplotlibJsPlugin
+
+
+def _init_theme():
+    # Set the Deephaven style globally.
+    # We use the savefig function to export the Figure, and that uses the Figure's properties for colours rather than temporary styling.
+    # The Figure's properties are set on creation time of the Figure, rather than when the Figure is exported
+    # We do not have hooks into when a user creates a new Figure, so we set the theme globally ahead of time
+    # https://github.com/matplotlib/matplotlib/issues/6592/
+    with resources.path(__package__, "deephaven.mplstyle") as p:
+        plt.style.use(["dark_background", p])
+
+
+class MatplotlibRegistration(Registration):
+    @classmethod
+    def register_into(cls, callback: Callback) -> None:
+        _init_theme()
+        plt.switch_backend("AGG")
+        from . import figure_type
+
+        callback = DheSafeCallbackWrapper(callback)
+
+        callback.register(figure_type.FigureType)
+
+        js_plugin = create_js_plugin(
+            PACKAGE_NAMESPACE,
+            JS_NAME,
+            PLUGIN_CLASS,
+        )
+
+        callback.register(js_plugin)

plugins/plotly-express/pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools>=43.0.0", "wheel"]
+requires = ["setuptools>=43.0.0", "wheel", "deephaven-plugin-packaging"]
 build-backend = "setuptools.build_meta"

plugins/plotly-express/setup.cfg

@@ -27,6 +27,7 @@ packages=find_namespace:
 install_requires =
     deephaven-plugin>=0.6.0
     plotly
+    deephaven-plugin-utilities
 include_package_data = True
 
 [options.packages.find]

plugins/plotly-express/setup.py

@@ -1,38 +1,10 @@
-import shutil
 from setuptools import setup
 import os
-import subprocess
-
-# Uses npm pack to create a tarball of the package and unpacks it into a build directory
-# Then uses that to add to the wheel
+from deephaven.plugin.packaging import package_js
 
 js_dir = "src/js/"
-dist_dir = os.path.join(js_dir, "dist")
-build_dir = os.path.join(js_dir, "build")
 dest_dir = os.path.join("src/deephaven/plot/express/_js")
-package_dir = os.path.join(build_dir, "package")
-
-# copy the bundle to the plotly-express directory
-# the path may not exist (e.g. when running tests)
-# so it is not strictly necessary to copy the bundle
-if os.path.exists(dist_dir):
-    # ignore errors as the directory may not exist
-    shutil.rmtree(build_dir, ignore_errors=True)
-    shutil.rmtree(dest_dir, ignore_errors=True)
-
-    os.makedirs(build_dir, exist_ok=True)
-
-    # pack and unpack into the js plotly-express directory
-    subprocess.run(
-        ["npm", "pack", "--pack-destination", "build"], cwd=js_dir, check=True
-    )
-    # it is assumed that there is only one tarball in the directory
-    files = os.listdir(build_dir)
-    for file in files:
-        subprocess.run(["tar", "-xzf", file], cwd=build_dir, check=True)
-        os.remove(os.path.join(build_dir, file))
 
-    # move the package directory to the expected package location
-    shutil.move(package_dir, dest_dir)
+package_js(js_dir, dest_dir)
 
 setup(package_data={"deephaven.plot.express._js": ["**"]})

plugins/plotly-express/src/deephaven/plot/express/_js_plugin.py

@@ -1,8 +1,4 @@
-import importlib.resources
-import json
 import pathlib
-import sys
-from typing import Callable, ContextManager
 
 from deephaven.plugin.js import JsPlugin
 
@@ -34,40 +30,3 @@ def main(self) -> str:
 
     def path(self) -> pathlib.Path:
         return self._path
-
-
-def _create_from_npm_package_json(
-    path_provider: Callable[[], ContextManager[pathlib.Path]]
-) -> JsPlugin:
-    with path_provider() as tmp_js_path:
-        js_path = tmp_js_path
-    if not js_path.exists():
-        raise Exception(
-            f"Package is not installed in a normal python filesystem, '{js_path}' does not exist"
-        )
-    with (js_path / "package.json").open("rb") as f:
-        package_json = json.load(f)
-    return ExpressJsPlugin(
-        package_json["name"],
-        package_json["version"],
-        package_json["main"],
-        js_path,
-    )
-
-
-def _resource_js_path() -> ContextManager[pathlib.Path]:
-    namespace = "deephaven.plot.express"
-    name = "_js"
-    if sys.version_info < (3, 9):
-        return importlib.resources.path(namespace, name)
-    else:
-        return importlib.resources.as_file(
-            importlib.resources.files(namespace).joinpath(name)
-        )
-
-
-def create_js_plugin() -> JsPlugin:
-    # TODO: Include developer instructions for installing in editable mode
-    # https://github.com/deephaven/deephaven-plugins/issues/93
-    # TBD what editable mode looks like for JsPlugin
-    return _create_from_npm_package_json(_resource_js_path)

plugins/plotly-express/src/deephaven/plot/express/_register.py

@@ -1,8 +1,11 @@
-import os
+from deephaven.plugin import Registration, Callback
+from deephaven.plugin.utilities import create_js_plugin, DheSafeCallbackWrapper
 from . import DeephavenFigureType
-from ._js_plugin import create_js_plugin
+from ._js_plugin import ExpressJsPlugin
 
-from deephaven.plugin import Registration, Callback
+PACKAGE_NAMESPACE = "deephaven.plot.express"
+JS_NAME = "_js"
+PLUGIN_CLASS = ExpressJsPlugin
 
 
 class ExpressRegistration(Registration):
@@ -21,8 +24,14 @@ def register_into(cls, callback: Callback) -> None:
             A function to call after registration
 
         """
+        callback = DheSafeCallbackWrapper(callback)
+
         callback.register(DeephavenFigureType)
 
-        # Only register the JS plugins if the environment variable is set
-        if os.getenv("DEEPHAVEN_ENABLE_PY_JS", "False").lower() in ("true", "1"):
-            callback.register(create_js_plugin())
+        js_plugin = create_js_plugin(
+            PACKAGE_NAMESPACE,
+            JS_NAME,
+            PLUGIN_CLASS,
+        )
+
+        callback.register(js_plugin)

plugins/ui/pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools>=43.0.0", "wheel"]
+requires = ["setuptools>=43.0.0", "wheel", "deephaven-plugin-packaging"]
 build-backend = "setuptools.build_meta"

plugins/ui/setup.cfg

@@ -3,7 +3,7 @@ name = deephaven-plugin-ui
 description = deephaven.ui plugin
 long_description = file: README.md
 long_description_content_type = text/markdown
-version = attr:deephaven.ui.__version__
+version = 0.9.0.dev0
 url = https://github.com/deephaven/deephaven-plugins
 project_urls =
     Source Code = https://github.com/deephaven/deephaven-plugins
@@ -28,6 +28,7 @@ install_requires =
     deephaven-core>=0.31.0
     deephaven-plugin>=0.6.0
     json-rpc
+    deephaven-plugin-utilities
     typing_extensions
 include_package_data = True
 
@@ -36,4 +37,4 @@ where=src
 
 [options.entry_points]
 deephaven.plugin =
-    registration_cls = deephaven.ui:UIRegistration
+    registration_cls = deephaven.ui._register:UIRegistration

plugins/ui/setup.py

@@ -0,0 +1,10 @@
+from setuptools import setup
+import os
+from deephaven.plugin.packaging import package_js
+
+js_dir = "src/js/"
+dest_dir = os.path.join("src/deephaven/ui/_js")
+
+package_js(js_dir, dest_dir)
+
+setup(package_data={"deephaven.ui._js": ["**"]})

plugins/ui/src/deephaven/ui/__init__.py

@@ -4,17 +4,7 @@
 The API is designed to be similar to React, but with some differences to make it more Pythonic.
 """
 
-from deephaven.plugin import Registration, Callback
 from .components import *
 from .elements import *
 from .hooks import *
 from .object_types import *
-
-__version__ = "0.9.0.dev0"
-
-
-class UIRegistration(Registration):
-    @classmethod
-    def register_into(cls, callback: Callback) -> None:
-        callback.register(DashboardType)
-        callback.register(ElementType)