Added a script to build deephaven-ib virtual environments.

.gitignore

@@ -1,5 +1,6 @@
 .idea
 venv
+venv-*
 build
 dist
 src/deephaven_ib.egg-info

dhib_env.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+
+""" A script to build a virtual environment for Deephaven-IB development or release."""
+
+import atexit
+import logging
+import os
+import shutil
+from pathlib import Path
+from types import ModuleType
+from typing import Optional, Dict, Union
+import click
+import pkginfo
+import requests
+
+########################################################################################################################
+# Shell
+########################################################################################################################
+
+
+def shell_exec(cmd: str) -> None:
+    """Execute a shell command.
+
+    Args:
+        cmd: The command to execute.
+    """
+    logging.warning(f"Executing shell command: {cmd}")
+    e = os.system(cmd)
+
+    if e != 0:
+        raise Exception(f"Error executing shell command: {cmd}")
+
+
+########################################################################################################################
+# Package Query Functions
+########################################################################################################################
+
+
+def delete_file_on_exit(file_path: Union[str, Path]) -> None:
+    """Register a file to be deleted on program exit."""
+
+    def delete_file():
+        if os.path.exists(file_path):
+            os.remove(file_path)
+            logging.debug(f"{file_path} has been deleted.")
+
+    atexit.register(delete_file)
+
+
+def download_wheel(python: str, package: str, version: Optional[str], delete_on_exit: bool = True) -> Path:
+    """Download a wheel file for a package with a specific version.
+
+    Args:
+        python: The path to the Python executable to use.
+        package: The name of the package to download.
+        version: The version of the package to download. If None, the latest version will be downloaded.
+        delete_on_exit: Whether to delete the wheel file on program exit.
+
+    Returns:
+        The path of the downloaded wheel file.
+
+    Raises:
+        subprocess.CalledProcessError: If the download process fails.
+    """
+    logging.warning(f"Downloading wheel for package: {package}, version: {version}, delete_on_exit: {delete_on_exit}")
+
+    if not version:
+        logging.warning(f"Determining latest version of package: {package}")
+        response = requests.get(f"https://pypi.org/pypi/{package}/json")
+        response.raise_for_status()
+        version = response.json()["info"]["version"]
+
+    ver = f"=={version}" if version else ""
+    shell_exec(f"{python} -m pip download {package}{ver} --no-deps")
+    p = Path(f"{package}-{version}-py3-none-any.whl").absolute()
+
+    if delete_on_exit:
+        delete_file_on_exit(str(p))
+
+    return p
+
+
+def pkg_dependencies(path_or_module: Union[str, Path, ModuleType]) -> Dict[str, Optional[str]]:
+    """Get the dependencies of a package.
+
+    Args:
+        path_or_module: The path to the package or the module object.
+
+    Returns:
+        A dictionary containing the dependencies of the package and their version specifications.
+    """
+
+    if isinstance(path_or_module, Path):
+        path_or_module = str(path_or_module)
+
+    meta = pkginfo.get_metadata(path_or_module)
+
+    if not meta:
+        raise ValueError(f"Package could not be found: {path_or_module}")
+
+    rst = {}
+
+    for req in meta.requires_dist:
+        s = req.split(" ")
+        name = s[0]
+
+        if len(s) > 1:
+            version = s[1].strip("()")
+        else:
+            version = None
+
+        rst[name] = version
+
+    return rst
+
+
+########################################################################################################################
+# Venv
+########################################################################################################################
+
+
+class Venv:
+    """A virtual environment."""
+
+    def __init__(self, is_release: bool, python: str, dh_version: str, ib_version: str, dh_ib_version: str,
+                 delete_if_exists: bool):
+        """Create a virtual environment.
+
+        Args:
+            is_release: Whether the virtual environment is for a release.
+            python: The path to the Python executable to use.
+            dh_version: The version of Deephaven.
+            ib_version: The version of ibapi.
+            dh_ib_version: The version of deephaven-ib.
+            delete_if_exists: Whether to delete the virtual environment if it already exists.
+        """
+        if is_release:
+            self.path = Path(f"venv-release-dhib={dh_version}").absolute()
+        else:
+            self.path = Path(f"venv-dev-dhib={dh_ib_version}-dh={dh_version}-ib={ib_version}").absolute()
+
+        logging.warning(f"Building new virtual environment: {self.path}")
+
+        if delete_if_exists and self.path.exists():
+            logging.warning(f"Deleting existing virtual environment: {self.path}")
+            shutil.rmtree(self.path)
+
+        if self.path.exists():
+            logging.error(f"Virtual environment already exists.  Please remove it before running this script. venv={self.path}")
+            raise FileExistsError(
+                f"Virtual environment already exists.  Please remove it before running this script. venv={self.path}")
+
+        logging.warning(f"Creating virtual environment: {self.path}")
+        shell_exec(f"{python} -m venv {self.path}")
+
+        logging.warning(f"Updating virtual environment: {self.path}")
+        shell_exec(f"{self.python} -m pip install --upgrade pip")
+        shell_exec(f"{self.python} -m pip install --upgrade build")
+
+    @property
+    def python(self) -> str:
+        """The path to the Python executable in the virtual environment."""
+        return os.path.join(self.path, "bin", "python")
+
+    def pip_install(self, package: Union[str, Path], version: Optional[str] = None) -> None:
+        """Install a package into the virtual environment.
+
+        Args:
+            package: The name of the package to install.
+            version: The version of the package to install. If None, the latest version will be installed.
+        """
+        logging.warning(f"Installing package in venv: {package}, version: {version}, venv: {self.path}")
+
+        if isinstance(package, Path):
+            package = package.absolute()
+
+        ver = f"=={version}" if version else ""
+        cmd = f"""{self.python} -m pip install {package}{ver}"""
+        shell_exec(cmd)
+
+
+########################################################################################################################
+# IB Wheel
+########################################################################################################################
+
+class IbWheel:
+    def __init__(self, version: str):
+        """Create an IB wheel.
+
+        Args:
+            version: The version of the IB wheel.
+        """
+        self.version = version
+
+    def build(self) -> None:
+        """Build the IB wheel."""
+        logging.warning(f"Building IB wheel: {self.version}")
+        shell_exec(f"cd ibwhl && IB_VERSION={self.version} docker-compose up --abort-on-container-exit")
+
+    def install(self, v: Venv) -> None:
+        """Install the IB wheel into a virtual environment.
+
+        Args:
+            v: The virtual environment to install the wheel into.
+        """
+        logging.warning(f"Installing IB wheel in venv: {self.version} {v.path}")
+        # Strip the 0 from the version number
+        mod_ver = self.version.replace(".0", ".")
+        v.pip_install(Path(f"ibwhl/dist/ibapi-{mod_ver}-py3-none-any.whl").absolute())
+
+
+########################################################################################################################
+# deephaven-ib
+########################################################################################################################
+
+class DhIbWheel:
+    def __init__(self, version: str, dh_version: str, ib_version: str):
+        """Create a deephaven-ib wheel.
+
+        Args:
+            version: The version of the deephaven-ib wheel.
+            dh_version: The version of Deephaven.
+            ib_version: The version of ibapi.
+        """
+        self.version = version
+        self.dh_version = dh_version
+        self.ib_version = ib_version
+
+    def build(self, v: Venv) -> None:
+        """Build the deephaven-ib wheel."""
+        logging.warning(f"Building deephaven-ib: {self.version}")
+        shell_exec(f"DH_IB_VERSION={self.version} DH_VERSION={self.dh_version} IB_VERSION={self.ib_version} {v.python} -m build --wheel")
+
+    def install(self, v: Venv) -> None:
+        """Install the deephaven-ib wheel into a virtual environment."""
+        logging.warning(f"Installing deephaven-ib in venv: {self.version} {v.path}")
+        v.pip_install(Path(f"dist/deephaven_ib-{self.version}-py3-none-any.whl").absolute())
+
+
+########################################################################################################################
+# Messages
+########################################################################################################################
+
+def success(v: Venv) -> None:
+    """Print a success message.
+
+    Args:
+        v: The virtual environment.
+    """
+    logging.warning(f"Success!  Virtual environment created: {v.path}")
+    logging.warning(f"Activate the virtual environment with: source {v.path}/bin/activate")
+    logging.warning(f"Deactivate the virtual environment with: deactivate")
+
+
+########################################################################################################################
+# Click CLI
+########################################################################################################################
+
+
+@click.group()
+def cli():
+    """A script to build Deephaven-IB virtual environments."""
+    pass
+
+
+@click.command()
+@click.option('--python', default="python3", help='The path to the Python executable to use.')
+@click.option('--dh_version', default="0.33.3", help='The version of Deephaven.')
+@click.option('--ib_version', default="10.19.04", help='The version of ibapi.')
+@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.')
+@click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.')
+def dev(python: str, dh_version: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool):
+    """Create a development environment."""
+    logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}")
+
+    use_dev = dh_ib_version is None
+
+    if dh_ib_version is None:
+        dh_ib_version = "0.0.0.dev0"
+
+    v = Venv(False, python, dh_version, ib_version, dh_ib_version, delete_venv)
+
+    ib_wheel = IbWheel(ib_version)
+    ib_wheel.build()
+    ib_wheel.install(v)
+
+    v.pip_install("deephaven-server", dh_version)
+
+    if use_dev:
+        logging.warning(f"Building deephaven-ib from source: {dh_ib_version}")
+        dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version)
+        dh_ib_wheel.build(v)
+        dh_ib_wheel.install(v)
+    else:
+        logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}")
+        logging.warning(f"*** INSTALLED deephaven-ib MAY BE INCONSISTENT WITH INSTALLED DEPENDENCIES ***")
+        v.pip_install("deephaven-ib", dh_ib_version)
+
+    success(v)
+
+
+@click.command()
+@click.option('--python', default="python3", help='The path to the Python executable to use.')
+@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.')
+@click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.')
+def release(python: str, dh_ib_version: Optional[str], delete_venv: bool):
+    """Create a release environment."""
+    logging.warning(f"Creating release environment: python={python} dh_ib_version={dh_ib_version}")
+
+    wheel = download_wheel(python, "deephaven-ib", dh_ib_version)
+    deps = pkg_dependencies(wheel)
+    ib_version = deps["ibapi"]
+    dh_version = deps["deephaven-server"]
+
+    v = Venv(True, python, dh_version, ib_version, dh_ib_version, delete_venv)
+
+    ib_wheel = IbWheel(ib_version)
+    ib_wheel.build()
+    ib_wheel.install(v)
+
+    logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}")
+    v.pip_install("deephaven-ib", dh_ib_version)
+    success(v)
+
+
+cli.add_command(dev)
+cli.add_command(release)
+
+if __name__ == '__main__':
+    cli()

requirements_dhib_env.txt

@@ -0,0 +1,3 @@
+pkginfo
+click
+requests