From 01573754df56a046244d3f1254f5f5e93442a924 Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 6 Sep 2024 14:22:48 -0500 Subject: [PATCH] feat: Add a cookiecutter build script (#807) Fixes #714 Adds a new script that is essentially a (slightly) lighter version of `tools/plugin_builder.py` to the widget template. Tested with commands in readme Somehow an earlier version snuck into #740 on accident, hence it being "modified" instead of new --- .../README.md | 58 ++- .../plugin_builder.py | 457 +++++++++++------- 2 files changed, 348 insertions(+), 167 deletions(-) diff --git a/templates/widget/{{ cookiecutter.python_project_name }}/README.md b/templates/widget/{{ cookiecutter.python_project_name }}/README.md index 0348d64d9..458631eb7 100644 --- a/templates/widget/{{ cookiecutter.python_project_name }}/README.md +++ b/templates/widget/{{ cookiecutter.python_project_name }}/README.md @@ -23,7 +23,61 @@ The JavaScript files have the following structure: Additionally, the `test` directory contains Python tests for the plugin. This demonstrates how the embedded Deephaven server can be used in tests. It's recommended to use `tox` to run the tests, and the `tox.ini` file is included in the project. -## Building the Plugin +## Using plugin_builder.py +The `plugin_builder.py` script is the recommended way to build the plugin. +See [Building the Plugin](#building-the-plugin) for more information if you want to build the plugin manually instead. + +To use `plugin_builder.py`, first set up your Python environment and install the required packages. +To build the plugin, you will need `npm` and `python` installed, as well as the `build` package for Python. +`nvm` is also strongly recommended, and an `.nvmrc` file is included in the project. +The script uses `watchdog` and `deephaven-server` for `--watch` mode and `--server` mode, respectively. +```sh +cd {{ cookiecutter.python_project_name }} +python -m venv .venv +source .venv/bin/activate +cd src/js +nvm install +npm install +cd ../.. +pip install --upgrade -r requirements.txt +pip install deephaven-server watchdog +``` + +First, run an initial install of the plugin: +This builds and installs the full plugin, including the JavaScript code. +```sh +python plugin_builder.py --install --js +``` + +After this, more advanced options can be used. +For example, if only iterating on the plugins with no version bumps, use the `--reinstall` flag for faster builds. +This adds `--force-reinstall --no-deps` to the `pip install` command. +```sh +python plugin_builder.py --reinstall --js +``` + +If only the Python code has changed, the `--js` flag can be omitted. +```sh +python plugin_builder.py --reinstall +``` + +Additional especially useful flags are `--watch` and `--server`. +`--watch` will watch the Python and JavaScript files for changes and rebuild the plugin when they are modified. +`--server` will start the Deephaven server with the plugin installed. +Taken in combination with `--reinstall` and `--js`, this command will rebuild and restart the server when changes are made to the plugin. +```sh +python plugin_builder.py --reinstall --js --watch --server +``` + +If interested in passing args to the server, the `--server-arg` flag can be used as well +Check `deephaven server --help` for more information on the available arguments. +```sh +python plugin_builder.py --reinstall --js --watch --server --server-arg --port=9999 +``` + +See [Using the Plugin](#using-the-plugin) for more information on how to use the plugin. + +## Manually Building the Plugin To build the plugin, you will need `npm` and `python` installed, as well as the `build` package for Python. `nvm` is also strongly recommended, and an `.nvmrc` file is included in the project. @@ -90,7 +144,7 @@ obj.send_message("Hello, world!") The panel can also send messages back to the Python client by using the input field. ## Debugging the Plugin -It's recommended to run through all the steps in Installing the Plugin and Using the Plugin to ensure the plugin is working correctly. +It's recommended to run through all the steps in [Using plugin_builder.py](#Using-plugin_builder.py) and [Using the Plugin](#Using-the-plugin) to ensure the plugin is working correctly. Then, make changes to the plugin and rebuild it to see the changes in action. Checkout the [Deephaven plugins repo](https://github.com/deephaven/deephaven-plugins), which is where this template was generated from, for more examples and information. The `plugins` folder contains current plugins that are developed and maintained by Deephaven. diff --git a/templates/widget/{{ cookiecutter.python_project_name }}/plugin_builder.py b/templates/widget/{{ cookiecutter.python_project_name }}/plugin_builder.py index a5813ee6f..8349153d5 100644 --- a/templates/widget/{{ cookiecutter.python_project_name }}/plugin_builder.py +++ b/templates/widget/{{ cookiecutter.python_project_name }}/plugin_builder.py @@ -3,54 +3,156 @@ import click import os import sys -from typing import Generator +from typing import Generator, Callable +import time +import subprocess +from watchdog.events import FileSystemEvent, RegexMatchingEventHandler +from watchdog.observers import Observer +import threading # get the directory of the current file +# this is used to watch for changes in this directory current_dir = os.path.dirname(os.path.abspath(__file__)) -# navigate out one directory to get to the plugins directory -plugins_dir = os.path.join(current_dir, "../plugins") - -def clean_build_dist(plugin: str) -> None: +# these are the patterns to watch for changes in this directory +# if in editable mode, the builder will rerun when these files change +REBUILD_REGEXES = [ + ".*\.py$", + ".*\.js$", + ".*\.ts$", + ".*\.tsx$", + ".*\.scss$", +] + +# ignore these patterns in particular +# prevents infinite loops when the builder is rerun +IGNORE_REGEXES = [ + ".*/dist/.*", + ".*/build/.*", + ".*/node_modules/.*", + ".*/_js/.*", + # ignore hidden files and directories + ".*/\..*/.*", +] + +# the path where the python files are located relative to this script +# modify this if the python files are moved +PYTHON_DIR = "." +# the path where the JS files are located relative to this script +# modify this if the JS files are moved +JS_DIR = "./src/js" + + +class PluginsChangedHandler(RegexMatchingEventHandler): """ - Remove the build and dist directories for a plugin. + A handler that watches for changes and reruns the function when changes are detected Args: - plugin: The plugin to clean. + func: The function to run when changes are detected + stop_event: The event to signal the function to stop - Returns: - None + Attributes: + func: The function to run when changes are detected + stop_event: The event to signal the function to stop + rerun_lock: A lock to prevent multiple reruns from occurring at the same time """ - # these folders may not exist, so ignore the errors - if os.path.exists(f"{plugins_dir}/{plugin}/build"): - os.system(f"rm -rf {plugins_dir}/{plugin}/build") - if os.path.exists(f"{plugins_dir}/{plugin}/dist"): - os.system(f"rm -rf {plugins_dir}/{plugin}/dist") + def __init__(self, func: Callable, stop_event: threading.Event) -> None: + super().__init__(regexes=REBUILD_REGEXES, ignore_regexes=IGNORE_REGEXES) -def plugin_names( - plugins: tuple[str], -) -> Generator[str, None, None]: - """ - Generate the plugins to use + self.func = func - Args: - plugins: The plugins to generate. If None, all plugins are yielded + # A flag to indicate whether the function should continue running + # Also prevents unnecessary reruns + self.stop_event = stop_event - Returns: - A generator of plugins + # A lock to prevent multiple reruns from occurring at the same time + self.rerun_lock = threading.Lock() + + # always have an initial run + threading.Thread(target=self.attempt_rerun).start() + + def attempt_rerun(self) -> None: + """ + Attempt to rerun the function. + If the stop event is set, do not rerun because a rerun has already been scheduled. + """ + self.stop_event.set() + with self.rerun_lock: + self.stop_event.clear() + self.func() + + def event_handler(self, event: FileSystemEvent) -> None: + """ + Handle any file system event + + Args: + event: The event that occurred + """ + if self.stop_event.is_set(): + # a rerun has already been scheduled on another thread + print( + f"File {event.src_path} {event.event_type}, rerun has already been scheduled" + ) + return + print(f"File {event.src_path} {event.event_type}, new rerun scheduled") + threading.Thread(target=self.attempt_rerun).start() + + def on_created(self, event: FileSystemEvent) -> None: + """ + Handle a file creation event + + Args: + event: The event that occurred + """ + self.event_handler(event) + + def on_deleted(self, event: FileSystemEvent) -> None: + """ + Handle a file deletion event + + Args: + event: The event that occurred + """ + self.event_handler(event) + + def on_modified(self, event: FileSystemEvent) -> None: + """ + Handle a file modification event + + Args: + event: The event that occurred + """ + self.event_handler(event) + + def on_moved(self, event: FileSystemEvent) -> None: + """ + Handle a file move event + + Args: + event: The event that occurred + + Returns: + + """ + self.event_handler(event) + + +def clean_build_dist() -> None: + """ + Remove the build and dist directories. """ - if plugins: - for plugin in plugins: - yield plugin - else: - for plugin in os.listdir(plugins_dir): - yield plugin + # these folders may not exist, so ignore the errors + if os.path.exists(f"{PYTHON_DIR}/build"): + os.system(f"rm -rf {PYTHON_DIR}/build") + if os.path.exists(f"{PYTHON_DIR}/dist"): + os.system(f"rm -rf {PYTHON_DIR}/dist") def run_command(command: str) -> None: """ Run a command and exit if it fails. + This should only be used in a non-main thread. Args: command: The command to run. @@ -60,44 +162,27 @@ def run_command(command: str) -> None: """ code = os.system(command) if code != 0: - sys.exit(1) + os._exit(1) -def run_build( - plugins: tuple[str], - error_on_missing: bool, -) -> None: +def run_build() -> None: """ - Build plugins that have a setup.cfg. - - Args: - plugins: The plugins to build. If None, all plugins with a setup.cfg are built. - error_on_missing: Whether to error if a plugin does not have a setup.cfg - - Returns: - None + Build the plugin """ - for plugin in plugin_names(plugins): - if os.path.exists(f"{plugins_dir}/{plugin}/setup.cfg"): - clean_build_dist(plugin) + clean_build_dist() - click.echo(f"Building {plugin}") - run_command(f"python -m build --wheel {plugins_dir}/{plugin}") - elif error_on_missing: - click.echo(f"Error: setup.cfg not found in {plugin}") - sys.exit(1) + click.echo(f"Building plugin") + run_command(f"python -m build --wheel {PYTHON_DIR}") def run_install( - plugins: tuple[str], reinstall: bool, ) -> None: """ Install plugins that have been built Args: - plugins: The plugins to install. If None, all plugins with a setup.cfg are installed. reinstall: Whether to reinstall the plugins. If True, the --force-reinstall and --no-deps flags are added to pip install. @@ -108,177 +193,219 @@ def run_install( if reinstall: install += " --force-reinstall --no-deps" - if plugins: - for plugin in plugins: - # a plugin would have failed in the build step if it didn't have a setup.cfg - click.echo(f"Installing {plugin}") - run_command(f"{install} {plugins_dir}/{plugin}/dist/*") - else: - click.echo("Installing all plugins") - run_command(f"{install} {plugins_dir}/*/dist/*") + click.echo("Installing plugin") + run_command(f"{install} {PYTHON_DIR}/dist/*.whl") -def run_docs( - plugins: tuple[str], - error_on_missing: bool, -) -> None: +def run_build_js() -> None: """ - Generate docs for plugins that have a make_docs.py - - Args: - plugins: The plugins to generate docs for. If None, all plugins with a make_docs.py are built. - error_on_missing: Whether to error if a plugin does not have a make_docs.py - - Returns: - None + Build the JS files for the plugin """ - for plugin in plugin_names(plugins): - if os.path.exists(f"{plugins_dir}/{plugin}/make_docs.py"): - click.echo(f"Generating docs for {plugin}") - run_command(f"python {plugins_dir}/{plugin}/make_docs.py") - elif error_on_missing: - click.echo(f"Error: make_docs.py not found in {plugin}") - sys.exit(1) + click.echo(f"Building the JS plugin") + run_command(f"npm run build --prefix {JS_DIR}") -def run_build_js(plugins: tuple[str]) -> None: +def build_server_args(server_arg: tuple[str]) -> list[str]: """ - Build the JS files for plugins that have a js directory + Build the server arguments to pass to the deephaven server + By default, the --no-browser flag is added to the server arguments unless the --browser flag is present Args: - plugins: The plugins to build. If None, all plugins with a js directory are built. - - Returns: - None + server_arg: The arguments to pass to the server """ - if plugins: - for plugin in plugins: - if os.path.exists(f"{plugins_dir}/{plugin}/src/js"): - click.echo(f"Building JS for {plugin}") - run_command(f"npm run build --prefix {plugins_dir}/{plugin}/src/js") - else: - click.echo(f"Error: src/js not found in {plugin}") - else: - click.echo(f"Building all JS plugins") - run_command(f"npm run build") - - -def run_configure( - configure: str | None, + server_args = ["--no-browser"] + if server_arg: + if "--no-browser" in server_arg or "--browser" in server_arg: + server_args = list(server_arg) + else: + server_args = server_args + list(server_arg) + return server_args + + +def handle_args( + build: bool, + install: bool, + reinstall: bool, + server: bool, + server_arg: tuple[str], + js: bool, + stop_event: threading.Event, ) -> None: """ - Configure the venv for plugin development + Handle all arguments for the builder command Args: - configure: The configuration to use. 'min' will install the minimum requirements for development. - 'full' will install some optional packages for development, such as sphinx and deephaven-server. - - Returns: - None + build: True to build the plugins + install: True to install the plugins + reinstall: True to reinstall the plugins + server: True to run the deephaven server after building and installing the plugins + server_arg: The arguments to pass to the server + js: True to build the JS files for the plugins + stop_event: The event to signal the function to stop """ - if configure in ["min", "full"]: - run_command("pip install -r requirements.txt") - run_command("pre-commit install") - run_command("npm install") - if configure == "full": - # currently deephaven-server is installed as part of the sphinx_ext requirements - run_command("pip install -r sphinx_ext/sphinx-requirements.txt") + # it is possible that the stop event is set before this function is called + if stop_event.is_set(): + return + + # default is to install, but don't if just configuring + if not any([build, install, reinstall, js]): + js = True + install = True + + # if this thread is signaled to stop, return after the current command + # instead of in the middle of a command, which could leave the environment in a bad state + if stop_event.is_set(): + return + + if js: + run_build_js() + + if stop_event.is_set(): + return + + if build or install or reinstall: + run_build() + + if stop_event.is_set(): + return + + if install or reinstall: + run_install(reinstall) + + if stop_event.is_set(): + return + + if server or server_arg: + server_args = build_server_args(server_arg) + + click.echo(f"Running deephaven server with args: {server_args}") + process = subprocess.Popen(["deephaven", "server"] + server_args) + + # waiting on either the process to finish or the stop event to be set + while not stop_event.wait(1): + poll = process.poll() + if poll is not None: + # process threw an error or was killed, so exit + os._exit(process.returncode) + + # stop event is set, so kill the process + process.terminate() + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + process.kill() + process.wait() @click.command( - short_help="Build and install plugins.", + short_help="Build and install plugin.", help="Build and install plugins. " "By default, all plugins with the necessary file are used unless specified via the plugins arg.", ) @click.option( - "--build", "-b", is_flag=True, help="Build all plugins that have a setup.cfg" + "--build", "-b", is_flag=True, help="Build the plugin." ) @click.option( "--install", "-i", is_flag=True, - help="Install all plugins that have a setup.cfg. This is the default behavior if no flags are provided.", + help="Install the plugin. This is the default behavior if no flags are provided.", ) @click.option( "--reinstall", "-r", is_flag=True, - help="Reinstall all plugins that have a setup.cfg. " + help="Reinstall the plugin. " "This adds the --force-reinstall and --no-deps flags to pip install. " - "Useful to reinstall a plugin that has already been installed and does not have a new version number.", -) -@click.option( - "--docs", - "-d", - is_flag=True, - help="Generate docs for all plugins that have a make_docs.py.", + "Useful if the plugin has already been installed and does not have a new version number.", ) @click.option( "--server", "-s", is_flag=True, - help="Run the deephaven server after building and installing the plugins.", + help="Run the deephaven server after building and installing the plugin.", +) +@click.option( + "--server-arg", + "-sa", + default=tuple(), + multiple=True, + help="Run the deephaven server after building and installing the plugin with the provided argument.", ) @click.option( "--js", "-j", is_flag=True, - help="Build the JS files for the plugins.", + help="Build the JS files for the plugin.", ) @click.option( - "--configure", - "-c", - default=None, - help="Configure your venv for plugin development. 'min' will install the minimum requirements for development." - "'full' will install some optional packages for development, such as sphinx and deephaven-server.", + "--watch", + "-w", + is_flag=True, + help="Run the other provided commands in an editable-like mode, watching for changes " + "This will rerun all other commands (except configure) when files are changed. " + "The top level directory of this project is watched.", ) -@click.argument("plugins", nargs=-1) def builder( build: bool, install: bool, reinstall: bool, server: bool, + server_arg: tuple[str], js: bool, - configure: str | None, - plugins: tuple[str], + watch: bool, ) -> None: """ Build and install plugins. Args: - build: True to build the plugins - install: True to install the plugins - reinstall: True to reinstall the plugins - docs: True to generate the docs - server: True to run the deephaven server after building and installing the plugins - js: True to build the JS files for the plugins - configure: The configuration to use. 'min' will install the minimum requirements for development. - 'full' will install some optional packages for development, such as sphinx and deephaven-server. - plugins: Plugins to build and install + build: True to build the plugin + install: True to install the plugin + reinstall: True to reinstall the plugin + server: True to run the deephaven server after building and installing the plugin + server_arg: The arguments to pass to the server + js: True to build the JS files for the plugin + watch: True to rerun the other commands when files are changed """ - run_configure(configure) - - # default is to install, but don't if just configuring - if not any([build, install, reinstall, docs, js, configure]): - js = True - install = True - - if js: - run_build_js(plugins) - - if build or install or reinstall: - run_build(plugins, len(plugins) > 0) - - if install or reinstall: - run_install(plugins, reinstall) - - if docs: - run_docs(plugins, len(plugins) > 0) - - if server: - click.echo("Running deephaven server") - os.system("deephaven server") + stop_event = threading.Event() + + def run_handle_args() -> None: + """ + Run the handle_args function with the provided arguments + """ + handle_args( + build, + install, + reinstall, + server, + server_arg, + js, + stop_event, + ) + + if not watch: + # since editable is not specified, only run the handler once + # call it from a thread to allow the usage of os._exit to exit the process + # rather than sys.exit because sys.exit will not exit the process when called from a thread + # and os._exit should be called from a thread + thread = threading.Thread(target=run_handle_args) + thread.start() + thread.join() + return + + # editable is specified, so run the handler in a loop that watches for changes and + # reruns the handler when changes are detected + event_handler = PluginsChangedHandler(run_handle_args, stop_event) + observer = Observer() + observer.schedule(event_handler, current_dir, recursive=True) + observer.start() + try: + while True: + input() + finally: + observer.stop() + observer.join() if __name__ == "__main__": - builder() + builder() \ No newline at end of file