From 15811a9fc3448aa7048ca7c8342927478b116c41 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Thu, 25 Apr 2024 14:57:22 -0600 Subject: [PATCH 01/51] DH version to 0.33.3 Java version to 17 --- .github/workflows/Dockerfile.pip | 4 ++-- .github/workflows/build-and-publish.yml | 6 +++--- docker/dev/Dockerfile | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Dockerfile.pip b/.github/workflows/Dockerfile.pip index 5eb27210..e0de4e5e 100644 --- a/.github/workflows/Dockerfile.pip +++ b/.github/workflows/Dockerfile.pip @@ -2,10 +2,10 @@ FROM ubuntu:22.04 RUN apt update && \ - apt install -y openjdk-11-jdk python3-pip python3-venv curl zip && \ + apt install -y openjdk-17-jdk python3-pip python3-venv curl zip && \ pip3 install --upgrade pip setuptools wheel -ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/ +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/ COPY ./wheels /wheels COPY ./ib-wheels /ib-wheels diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index c6d7318e..af5b3345 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -13,7 +13,7 @@ on: env: IB_VERSION: 10.19.01 - DH_VERSION: 0.28.1 + DH_VERSION: 0.33.3 jobs: build-ib-whl: @@ -114,7 +114,7 @@ jobs: - name: Apt installs run: | sudo apt update - sudo apt install -y openjdk-11-jdk + sudo apt install -y openjdk-17-jdk - name: Pip installs run: pip3 install --upgrade sphinx==4.2.0 sphinx-autodoc-typehints furo==2021.10.9 - name: Download IB wheels @@ -132,7 +132,7 @@ jobs: - name: Run Sphinx working-directory: ./sphinx env: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 run: | make html touch build/html/.nojekyll diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 18351e1f..4b0fa0d4 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -11,12 +11,12 @@ ARG DH_VERSION # Install requirements RUN apt update && \ - apt install -y openjdk-11-jdk && \ - ln -s /usr/lib/jvm/java-11-openjdk-*/ /usr/lib/jvm/java-11-openjdk && \ + apt install -y openjdk-17-jdk && \ + ln -s /usr/lib/jvm/java-17-openjdk-*/ /usr/lib/jvm/java-17-openjdk && \ apt install --yes git python3-venv python3-pip curl unzip && \ pip3 install --upgrade pip setuptools wheel build -ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk # Build and install ibapi From 780185e097fb3198357f55995409502419e80780 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Thu, 25 Apr 2024 15:27:51 -0600 Subject: [PATCH 02/51] Updated the ib api version to 10.19.04 --- .github/workflows/build-and-publish.yml | 2 +- README.md | 4 ++-- docker/dev/build.sh | 2 +- docker/release/docker-compose.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index af5b3345..eb4b27b7 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -12,7 +12,7 @@ on: - released env: - IB_VERSION: 10.19.01 + IB_VERSION: 10.19.04 DH_VERSION: 0.33.3 jobs: diff --git a/README.md b/README.md index a23c4022..fcfba5f8 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ new feature and has not been well tested. To do this: 1) Install `ibapi`: ```bash # pip installed version of ibapi is too old. You must download and install a more recent version. - export IB_VERSION=1019.01 + export IB_VERSION=1019.04 curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" unzip api.zip cd ./IBJts/source/pythonclient @@ -690,7 +690,7 @@ print(ibapi.__version__) If your version is `9.x`, it is the old version from [PyPI](https://pypi.org/project/ibapi/). To install the required `ibapi` version: ```bash # pip installed version of ibapi is too old. You must download and install a more recent version. -export IB_VERSION=1019.01 +export IB_VERSION=1019.04 curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" unzip api.zip cd ./IBJts/source/pythonclient diff --git a/docker/dev/build.sh b/docker/dev/build.sh index a3295363..9f00ef28 100755 --- a/docker/dev/build.sh +++ b/docker/dev/build.sh @@ -16,7 +16,7 @@ if [ -z "$DH_VERSION" ]; then exit 1 fi -IB_VERSION_DEFAULT=10.19.01 +IB_VERSION_DEFAULT=10.19.04 if [ -z "$IB_VERSION" ]; then echo "Using default IB_VERSION=${IB_VERSION_DEFAULT}" diff --git a/docker/release/docker-compose.yml b/docker/release/docker-compose.yml index 8b60e9a7..dea607e5 100644 --- a/docker/release/docker-compose.yml +++ b/docker/release/docker-compose.yml @@ -6,7 +6,7 @@ services: environment: # Deephaven PSK password: See https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/#setting-your-own-key DH_PASSWORD: ${DH_PASSWORD:?DH_PASSWORD not set} - command: python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g',f'-Dauthentication.psk={os.getenv(\"DH_PASSWORD\")}']); _server.start()" + command: python3 -i -c "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g',f'-Dauthentication.psk={os.getenv(\"DH_PASSWORD\")}']); _server.start()" stdin_open: true tty: true ports: From 65b9ea9a8862c4dd93cbc98172898fbbbaa6a177 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Mon, 29 Apr 2024 13:13:00 -0600 Subject: [PATCH 03/51] Streamline and simplify the IB artifact building. --- .github/workflows/build-and-publish.yml | 20 ++++------------ .gitignore | 1 + ibwhl/README.md | 30 +++++++++++++++++++++++ ibwhl/build_ibpy.sh | 32 +++++++++++++++++++++++++ ibwhl/docker-compose.yml | 29 ++++++++++++++++++++++ 5 files changed, 96 insertions(+), 16 deletions(-) create mode 100644 ibwhl/README.md create mode 100755 ibwhl/build_ibpy.sh create mode 100644 ibwhl/docker-compose.yml diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index eb4b27b7..0d8a8b4d 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -20,28 +20,16 @@ jobs: name: Build IB WHL runs-on: ubuntu-20.04 steps: - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel build twine + - name: Checkout + uses: actions/checkout@v1 - name: Build - run: | - IB_VERSION_DOWNLOAD=$(echo ${IB_VERSION} | sed 's/[.]//') - echo "Downloading IB API version ${IB_VERSION_DOWNLOAD}" - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION_DOWNLOAD}.zip" - unzip api.zip - cd ./IBJts/source/pythonclient - python -m build + run: cd ibwhl && docker-compose -f "docker-compose.yml" up -d --build - name: Archive build artifacts uses: actions/upload-artifact@v2 with: name: ib-wheels path: | - ./IBJts/source/pythonclient/dist/* + ibwhl/dist/* build-whl: diff --git a/.gitignore b/.gitignore index 73b04775..3c257aff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea venv +build dist src/deephaven_ib.egg-info docker/data diff --git a/ibwhl/README.md b/ibwhl/README.md new file mode 100644 index 00000000..0d89433c --- /dev/null +++ b/ibwhl/README.md @@ -0,0 +1,30 @@ +# Interactive Brokers Python Wheels Builder + +This directory provides a Docker Compose setup for building Python wheels for the Interactive Brokers API. + +## Requirements + +- Docker +- Docker Compose + +## Usage + +1. Set the `IB_VERSION` environment variable to the version of Interactive Brokers' API you want to use. +2. Run `docker compose up` from the directory containing the Docker Compose file. This will build the wheels. +3. After the build process is complete, the wheels will be available in the `./dist` directory. +4. To install the wheels, run `pip install ./dist/*.whl`. + +> **Note:** Steps 1 and 2 can be combined as `IB_VERSION=10.19.04 docker compose up`. + +## Details + +The build process is defined in the `build_ibpy.sh` script. +The `./build` directory contains the scratch work for building the wheels. + +By using Docker, you can build the wheels in a clean environment that is isolated from your local machine. +This setup will work on any platform that supports Docker. + +## Note + +Interactive Brokers does not make their Python wheels available via PyPI, and the wheels are not redistributable. +This script lets you build the wheels locally, and then install them via pip. \ No newline at end of file diff --git a/ibwhl/build_ibpy.sh b/ibwhl/build_ibpy.sh new file mode 100755 index 00000000..9512a743 --- /dev/null +++ b/ibwhl/build_ibpy.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# This script is used to build Python wheels for the Interactive Brokers' API. + + +# check that IB_VERSION is set +if [ -z "$IB_VERSION" ]; then + echo "IB_VERSION must be set" + exit 1 +fi + +# IB has a funky versioning scheme, so we need to strip out the periods in some places before download +IB_VERSION_DOWNLOAD=$(echo ${IB_VERSION} | sed 's/[.]//') + +rm -rf ./build +rm -rf ./dist + +mkdir ./build +mkdir ./dist + +pushd ./build + +python3 -m pip install build + +echo "Downloading IB API version ${IB_VERSION} (${IB_VERSION_DOWNLOAD})" +curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION_DOWNLOAD}.zip" +unzip api.zip +cd ./IBJts/source/pythonclient +python3 -m build --wheel +popd +cp ./build/IBJts/source/pythonclient/dist/* ./dist/ + diff --git a/ibwhl/docker-compose.yml b/ibwhl/docker-compose.yml new file mode 100644 index 00000000..d37a910f --- /dev/null +++ b/ibwhl/docker-compose.yml @@ -0,0 +1,29 @@ + +# This is a Docker Compose file for building Interactive Brokers python wheels. +# Ideally, the wheels would be published by IB for installation via pip, but +# (1) IB does not make them available via pypi, and +# (2) the wheels are not redistributable. +# This script lets you build the wheels locally, and then install them via pip. +# +# To build the wheels, run `docker-compose up` from the directory containing this file. +# To install the wheels, run `pip install ./dist/*.whl`. +# After running the script, the wheels will be available in the `./dist` directory. +# The `./build` directory contains the scratch work for building the wheels. +# +# The build process is defined in the `build_ibpy.sh` script. +# +# By using Docker, you can build the wheels in a clean environment that is isolated from your local machine. +# It also will work on any platform that supports Docker. + +version: '3.8' + +services: + build: + image: python:3.10 + environment: + - IB_VERSION + volumes: + - .:/app + - ./build:/build + - ./dist:/dist + command: /bin/bash -c "/app/build_ibpy.sh" \ No newline at end of file From 4d8f657e3398f73b2c212bc60503f7e652c59f3f Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 13:40:19 -0600 Subject: [PATCH 04/51] Added a script to build deephaven-ib virtual environments. --- .gitignore | 1 + dhib_env.py | 330 ++++++++++++++++++++++++++++++++++++++ requirements_dhib_env.txt | 3 + 3 files changed, 334 insertions(+) create mode 100755 dhib_env.py create mode 100644 requirements_dhib_env.txt diff --git a/.gitignore b/.gitignore index 3c257aff..28216ca7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea venv +venv-* build dist src/deephaven_ib.egg-info diff --git a/dhib_env.py b/dhib_env.py new file mode 100755 index 00000000..e60f544f --- /dev/null +++ b/dhib_env.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 + +""" A script to build a virtual environment for Deephaven-IB development or release.""" + +import atexit +import logging +import os +import shutil +from pathlib import Path +from types import ModuleType +from typing import Optional, Dict, Union +import click +import pkginfo +import requests + +######################################################################################################################## +# Shell +######################################################################################################################## + + +def shell_exec(cmd: str) -> None: + """Execute a shell command. + + Args: + cmd: The command to execute. + """ + logging.warning(f"Executing shell command: {cmd}") + e = os.system(cmd) + + if e != 0: + raise Exception(f"Error executing shell command: {cmd}") + + +######################################################################################################################## +# Package Query Functions +######################################################################################################################## + + +def delete_file_on_exit(file_path: Union[str, Path]) -> None: + """Register a file to be deleted on program exit.""" + + def delete_file(): + if os.path.exists(file_path): + os.remove(file_path) + logging.debug(f"{file_path} has been deleted.") + + atexit.register(delete_file) + + +def download_wheel(python: str, package: str, version: Optional[str], delete_on_exit: bool = True) -> Path: + """Download a wheel file for a package with a specific version. + + Args: + python: The path to the Python executable to use. + package: The name of the package to download. + version: The version of the package to download. If None, the latest version will be downloaded. + delete_on_exit: Whether to delete the wheel file on program exit. + + Returns: + The path of the downloaded wheel file. + + Raises: + subprocess.CalledProcessError: If the download process fails. + """ + logging.warning(f"Downloading wheel for package: {package}, version: {version}, delete_on_exit: {delete_on_exit}") + + if not version: + logging.warning(f"Determining latest version of package: {package}") + response = requests.get(f"https://pypi.org/pypi/{package}/json") + response.raise_for_status() + version = response.json()["info"]["version"] + + ver = f"=={version}" if version else "" + shell_exec(f"{python} -m pip download {package}{ver} --no-deps") + p = Path(f"{package}-{version}-py3-none-any.whl").absolute() + + if delete_on_exit: + delete_file_on_exit(str(p)) + + return p + + +def pkg_dependencies(path_or_module: Union[str, Path, ModuleType]) -> Dict[str, Optional[str]]: + """Get the dependencies of a package. + + Args: + path_or_module: The path to the package or the module object. + + Returns: + A dictionary containing the dependencies of the package and their version specifications. + """ + + if isinstance(path_or_module, Path): + path_or_module = str(path_or_module) + + meta = pkginfo.get_metadata(path_or_module) + + if not meta: + raise ValueError(f"Package could not be found: {path_or_module}") + + rst = {} + + for req in meta.requires_dist: + s = req.split(" ") + name = s[0] + + if len(s) > 1: + version = s[1].strip("()") + else: + version = None + + rst[name] = version + + return rst + + +######################################################################################################################## +# Venv +######################################################################################################################## + + +class Venv: + """A virtual environment.""" + + def __init__(self, is_release: bool, python: str, dh_version: str, ib_version: str, dh_ib_version: str, + delete_if_exists: bool): + """Create a virtual environment. + + Args: + is_release: Whether the virtual environment is for a release. + python: The path to the Python executable to use. + dh_version: The version of Deephaven. + ib_version: The version of ibapi. + dh_ib_version: The version of deephaven-ib. + delete_if_exists: Whether to delete the virtual environment if it already exists. + """ + if is_release: + self.path = Path(f"venv-release-dhib={dh_version}").absolute() + else: + self.path = Path(f"venv-dev-dhib={dh_ib_version}-dh={dh_version}-ib={ib_version}").absolute() + + logging.warning(f"Building new virtual environment: {self.path}") + + if delete_if_exists and self.path.exists(): + logging.warning(f"Deleting existing virtual environment: {self.path}") + shutil.rmtree(self.path) + + if self.path.exists(): + logging.error(f"Virtual environment already exists. Please remove it before running this script. venv={self.path}") + raise FileExistsError( + f"Virtual environment already exists. Please remove it before running this script. venv={self.path}") + + logging.warning(f"Creating virtual environment: {self.path}") + shell_exec(f"{python} -m venv {self.path}") + + logging.warning(f"Updating virtual environment: {self.path}") + shell_exec(f"{self.python} -m pip install --upgrade pip") + shell_exec(f"{self.python} -m pip install --upgrade build") + + @property + def python(self) -> str: + """The path to the Python executable in the virtual environment.""" + return os.path.join(self.path, "bin", "python") + + def pip_install(self, package: Union[str, Path], version: Optional[str] = None) -> None: + """Install a package into the virtual environment. + + Args: + package: The name of the package to install. + version: The version of the package to install. If None, the latest version will be installed. + """ + logging.warning(f"Installing package in venv: {package}, version: {version}, venv: {self.path}") + + if isinstance(package, Path): + package = package.absolute() + + ver = f"=={version}" if version else "" + cmd = f"""{self.python} -m pip install {package}{ver}""" + shell_exec(cmd) + + +######################################################################################################################## +# IB Wheel +######################################################################################################################## + +class IbWheel: + def __init__(self, version: str): + """Create an IB wheel. + + Args: + version: The version of the IB wheel. + """ + self.version = version + + def build(self) -> None: + """Build the IB wheel.""" + logging.warning(f"Building IB wheel: {self.version}") + shell_exec(f"cd ibwhl && IB_VERSION={self.version} docker-compose up --abort-on-container-exit") + + def install(self, v: Venv) -> None: + """Install the IB wheel into a virtual environment. + + Args: + v: The virtual environment to install the wheel into. + """ + logging.warning(f"Installing IB wheel in venv: {self.version} {v.path}") + # Strip the 0 from the version number + mod_ver = self.version.replace(".0", ".") + v.pip_install(Path(f"ibwhl/dist/ibapi-{mod_ver}-py3-none-any.whl").absolute()) + + +######################################################################################################################## +# deephaven-ib +######################################################################################################################## + +class DhIbWheel: + def __init__(self, version: str, dh_version: str, ib_version: str): + """Create a deephaven-ib wheel. + + Args: + version: The version of the deephaven-ib wheel. + dh_version: The version of Deephaven. + ib_version: The version of ibapi. + """ + self.version = version + self.dh_version = dh_version + self.ib_version = ib_version + + def build(self, v: Venv) -> None: + """Build the deephaven-ib wheel.""" + logging.warning(f"Building deephaven-ib: {self.version}") + shell_exec(f"DH_IB_VERSION={self.version} DH_VERSION={self.dh_version} IB_VERSION={self.ib_version} {v.python} -m build --wheel") + + def install(self, v: Venv) -> None: + """Install the deephaven-ib wheel into a virtual environment.""" + logging.warning(f"Installing deephaven-ib in venv: {self.version} {v.path}") + v.pip_install(Path(f"dist/deephaven_ib-{self.version}-py3-none-any.whl").absolute()) + + +######################################################################################################################## +# Messages +######################################################################################################################## + +def success(v: Venv) -> None: + """Print a success message. + + Args: + v: The virtual environment. + """ + logging.warning(f"Success! Virtual environment created: {v.path}") + logging.warning(f"Activate the virtual environment with: source {v.path}/bin/activate") + logging.warning(f"Deactivate the virtual environment with: deactivate") + + +######################################################################################################################## +# Click CLI +######################################################################################################################## + + +@click.group() +def cli(): + """A script to build Deephaven-IB virtual environments.""" + pass + + +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--dh_version', default="0.33.3", help='The version of Deephaven.') +@click.option('--ib_version', default="10.19.04", help='The version of ibapi.') +@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') +def dev(python: str, dh_version: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool): + """Create a development environment.""" + logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}") + + use_dev = dh_ib_version is None + + if dh_ib_version is None: + dh_ib_version = "0.0.0.dev0" + + v = Venv(False, python, dh_version, ib_version, dh_ib_version, delete_venv) + + ib_wheel = IbWheel(ib_version) + ib_wheel.build() + ib_wheel.install(v) + + v.pip_install("deephaven-server", dh_version) + + if use_dev: + logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") + dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version) + dh_ib_wheel.build(v) + dh_ib_wheel.install(v) + else: + logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") + logging.warning(f"*** INSTALLED deephaven-ib MAY BE INCONSISTENT WITH INSTALLED DEPENDENCIES ***") + v.pip_install("deephaven-ib", dh_ib_version) + + success(v) + + +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') +def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): + """Create a release environment.""" + logging.warning(f"Creating release environment: python={python} dh_ib_version={dh_ib_version}") + + wheel = download_wheel(python, "deephaven-ib", dh_ib_version) + deps = pkg_dependencies(wheel) + ib_version = deps["ibapi"] + dh_version = deps["deephaven-server"] + + v = Venv(True, python, dh_version, ib_version, dh_ib_version, delete_venv) + + ib_wheel = IbWheel(ib_version) + ib_wheel.build() + ib_wheel.install(v) + + logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") + v.pip_install("deephaven-ib", dh_ib_version) + success(v) + + +cli.add_command(dev) +cli.add_command(release) + +if __name__ == '__main__': + cli() diff --git a/requirements_dhib_env.txt b/requirements_dhib_env.txt new file mode 100644 index 00000000..35cc9f4b --- /dev/null +++ b/requirements_dhib_env.txt @@ -0,0 +1,3 @@ +pkginfo +click +requests From 482a2bbaa2bb88631e4b8b2358d1b9aa0fc70e74 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 13:41:47 -0600 Subject: [PATCH 05/51] Made IB wheel building fail on errors. --- ibwhl/build_ibpy.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ibwhl/build_ibpy.sh b/ibwhl/build_ibpy.sh index 9512a743..36a791f5 100755 --- a/ibwhl/build_ibpy.sh +++ b/ibwhl/build_ibpy.sh @@ -2,6 +2,8 @@ # This script is used to build Python wheels for the Interactive Brokers' API. +# Exit script on first error +set -e # check that IB_VERSION is set if [ -z "$IB_VERSION" ]; then @@ -12,11 +14,16 @@ fi # IB has a funky versioning scheme, so we need to strip out the periods in some places before download IB_VERSION_DOWNLOAD=$(echo ${IB_VERSION} | sed 's/[.]//') -rm -rf ./build -rm -rf ./dist +if [ ! -d "./build" ]; then + mkdir ./build +fi + +if [ ! -d "./dist" ]; then + mkdir ./dist +fi -mkdir ./build -mkdir ./dist +rm -rf ./build/* +rm -rf ./dist/* pushd ./build From 527e3d779e77d9ad3ce9d1a95bb3bf4889843c45 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 13:47:49 -0600 Subject: [PATCH 06/51] Updated to server-ui deephaven images. --- .github/workflows/Dockerfile.server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Dockerfile.server b/.github/workflows/Dockerfile.server index 24ed1d85..9f3d288f 100644 --- a/.github/workflows/Dockerfile.server +++ b/.github/workflows/Dockerfile.server @@ -1,6 +1,6 @@ ARG DH_VERSION -FROM ghcr.io/deephaven/server:${DH_VERSION} +FROM ghcr.io/deephaven/server-ui:${DH_VERSION} RUN apt update && \ apt install -y python3-pip python3-venv curl zip && \ From bdc638d0902296e12ea214836e32b5f0f8cdecc2 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 13:54:50 -0600 Subject: [PATCH 07/51] Debug CI --- .github/workflows/build-and-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 0d8a8b4d..2f4a05f4 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -24,6 +24,9 @@ jobs: uses: actions/checkout@v1 - name: Build run: cd ibwhl && docker-compose -f "docker-compose.yml" up -d --build + # TODO: remove + - name: DEBUG + run: pwd && ls -la ibwhl/dist - name: Archive build artifacts uses: actions/upload-artifact@v2 with: From 6a8781d48eb1710aef18804545715f1c8a9e1116 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 13:57:49 -0600 Subject: [PATCH 08/51] Debug CI --- .github/workflows/build-and-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 2f4a05f4..f01651e5 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -22,6 +22,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v1 + # TODO: remove + - name: DEBUG + run: pwd - name: Build run: cd ibwhl && docker-compose -f "docker-compose.yml" up -d --build # TODO: remove From 91972eaeaf6b8ca29c04741c9affd1a2d621a36a Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 13:59:55 -0600 Subject: [PATCH 09/51] Debug CI --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index f01651e5..1fd3e7ca 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -29,7 +29,7 @@ jobs: run: cd ibwhl && docker-compose -f "docker-compose.yml" up -d --build # TODO: remove - name: DEBUG - run: pwd && ls -la ibwhl/dist + run: pwd && find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 with: From 5d1525d6552343befbfe959a0784211a991f2ea3 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 14:02:21 -0600 Subject: [PATCH 10/51] Debug CI --- .github/workflows/build-and-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 1fd3e7ca..32a6f39f 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -26,7 +26,10 @@ jobs: - name: DEBUG run: pwd - name: Build - run: cd ibwhl && docker-compose -f "docker-compose.yml" up -d --build + run: | + cd ibwhl + docker-compose -f "docker-compose.yml" up --build + find . -name \*.whl # TODO: remove - name: DEBUG run: pwd && find . -name \*.whl From f906127ce1317557347a842516a7b410f7b4190b Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 14:04:29 -0600 Subject: [PATCH 11/51] Removed the server-ui change since it requires 0.34 --- .github/workflows/Dockerfile.server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Dockerfile.server b/.github/workflows/Dockerfile.server index 9f3d288f..24ed1d85 100644 --- a/.github/workflows/Dockerfile.server +++ b/.github/workflows/Dockerfile.server @@ -1,6 +1,6 @@ ARG DH_VERSION -FROM ghcr.io/deephaven/server-ui:${DH_VERSION} +FROM ghcr.io/deephaven/server:${DH_VERSION} RUN apt update && \ apt install -y python3-pip python3-venv curl zip && \ From ba169a39934a19bc1b6a6984c7d2e376c192b116 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 14:17:41 -0600 Subject: [PATCH 12/51] Made the versions easier to update. --- .github/workflows/build-and-publish.yml | 6 ------ dhib_env.py | 7 +++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 32a6f39f..c335ecfb 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -22,17 +22,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v1 - # TODO: remove - - name: DEBUG - run: pwd - name: Build run: | cd ibwhl docker-compose -f "docker-compose.yml" up --build find . -name \*.whl - # TODO: remove - - name: DEBUG - run: pwd && find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 with: diff --git a/dhib_env.py b/dhib_env.py index e60f544f..f43e3579 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -13,6 +13,9 @@ import pkginfo import requests +IB_VERSION_DEFAULT="10.19.04" +DH_VERSION_DEFAULT="0.33.3" + ######################################################################################################################## # Shell ######################################################################################################################## @@ -265,8 +268,8 @@ def cli(): @click.command() @click.option('--python', default="python3", help='The path to the Python executable to use.') -@click.option('--dh_version', default="0.33.3", help='The version of Deephaven.') -@click.option('--ib_version', default="10.19.04", help='The version of ibapi.') +@click.option('--dh_version', default=DH_VERSION_DEFAULT, help='The version of Deephaven.') +@click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') @click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') @click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') def dev(python: str, dh_version: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool): From 9be1e6c32a604699be5136de507a082ac1ab9c73 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 14:18:17 -0600 Subject: [PATCH 13/51] Remove legacy docker --- docker/dev/Dockerfile | 43 ------------------------------- docker/dev/README.md | 27 ------------------- docker/dev/build.sh | 28 -------------------- docker/dev/docker-compose.yml | 17 ------------ docker/release/README.md | 22 ---------------- docker/release/docker-compose.yml | 17 ------------ 6 files changed, 154 deletions(-) delete mode 100644 docker/dev/Dockerfile delete mode 100644 docker/dev/README.md delete mode 100755 docker/dev/build.sh delete mode 100644 docker/dev/docker-compose.yml delete mode 100644 docker/release/README.md delete mode 100644 docker/release/docker-compose.yml diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile deleted file mode 100644 index 4b0fa0d4..00000000 --- a/docker/dev/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# -# A docker image from the current repository. -# - -FROM ubuntu:22.04 - -ARG DH_IB_VERSION=0.0.0dev -ARG IB_VERSION -ARG DH_VERSION - -# Install requirements - -RUN apt update && \ - apt install -y openjdk-17-jdk && \ - ln -s /usr/lib/jvm/java-17-openjdk-*/ /usr/lib/jvm/java-17-openjdk && \ - apt install --yes git python3-venv python3-pip curl unzip && \ - pip3 install --upgrade pip setuptools wheel build - -ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk - -# Build and install ibapi - -RUN mkdir /build && \ - cd /build && \ - IB_VERSION_DOWNLOAD=$(echo ${IB_VERSION} | sed 's/[.]//') && \ - echo "Downloading IB API version ${IB_VERSION_DOWNLOAD}" && \ - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION_DOWNLOAD}.zip" && \ - unzip api.zip && \ - cd ./IBJts/source/pythonclient && \ - python3 setup.py install && \ - cd / && \ - rm -rf /build - -# Build and install deephaven-ib - -COPY ./build /build - -RUN cd /build && \ - python3 -m build && \ - pip3 install dist/*.whl && \ - rm -rf /build - -CMD python3 diff --git a/docker/dev/README.md b/docker/dev/README.md deleted file mode 100644 index 67d6194f..00000000 --- a/docker/dev/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Build Docker Image From Current Deephaven-IB Checkout - -This directory contains the ingredients to build Docker images from the current, checked-out deephaven-ib repository. - -This is useful when doing local development or when official images are not available for your platform. - -In general, you will want to use the officially released images at [https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib](https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib). - -## Build Image - -```bash -./build.sh -``` - -## Run image in interactive mode - -```bash -docker compose up -``` - -or - -```bash -# Set jvm_args to the desired JVM memory for Deephaven -docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 deephaven-examples/deephaven-ib:dev python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()" -``` - diff --git a/docker/dev/build.sh b/docker/dev/build.sh deleted file mode 100755 index 9f00ef28..00000000 --- a/docker/dev/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Build a docker image from the current repository. -# - -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd ${__dir} - -rm -rf build -mkdir build -rsync -av ../.. build --exclude docker -rm -rf build/dist - -if [ -z "$DH_VERSION" ]; then - echo "DH_VERSION must be set" - exit 1 -fi - -IB_VERSION_DEFAULT=10.19.04 - -if [ -z "$IB_VERSION" ]; then - echo "Using default IB_VERSION=${IB_VERSION_DEFAULT}" - IB_VERSION=${IB_VERSION_DEFAULT} -fi - -docker build --build-arg DH_VERSION=${DH_VERSION} --build-arg IB_VERSION=${IB_VERSION} -t deephaven-examples/deephaven-ib:dev -f Dockerfile . - -rm -rf build \ No newline at end of file diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml deleted file mode 100644 index 818ec6a6..00000000 --- a/docker/dev/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.4" - -services: - server: - image: deephaven-examples/deephaven-ib:dev - environment: - # Deephaven PSK password: See https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/#setting-your-own-key - DH_PASSWORD: ${DH_PASSWORD:?DH_PASSWORD not set} - command: python3 -i -c "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g',f'-Dauthentication.psk={os.getenv(\"DH_PASSWORD\")}']); _server.start()" - stdin_open: true - tty: true - ports: - - "${DEEPHAVEN_PORT:-10000}:10000" - volumes: - - ./data:/data - - ~/.deephaven:/storage - diff --git a/docker/release/README.md b/docker/release/README.md deleted file mode 100644 index b5fe0069..00000000 --- a/docker/release/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Build Docker Image From The Most Recent pip-installable Deephaven-IB Release - -This directory contains the ingredients to launch a prebuilt deephaven-ib image. - -In general, you will want to use the officially released images at [https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib](https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib). - -## Build Image - -```bash -./build.sh -``` - -## Run image in interactive mode - -```bash -DH_IB_VERSION= docker compose up -``` - -```bash -# Set jvm_args to the desired JVM memory for Deephaven -docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 deephaven-examples/deephaven-ib: python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()" -``` diff --git a/docker/release/docker-compose.yml b/docker/release/docker-compose.yml deleted file mode 100644 index dea607e5..00000000 --- a/docker/release/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.4" - -services: - server: - image: ghcr.io/deephaven-examples/deephaven-ib:${DH_IB_VERSION:?DH_IB_VERSION not set} - environment: - # Deephaven PSK password: See https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/#setting-your-own-key - DH_PASSWORD: ${DH_PASSWORD:?DH_PASSWORD not set} - command: python3 -i -c "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g',f'-Dauthentication.psk={os.getenv(\"DH_PASSWORD\")}']); _server.start()" - stdin_open: true - tty: true - ports: - - "${DEEPHAVEN_PORT:-10000}:10000" - volumes: - - ./data:/data - - ~/.deephaven:/storage - From b82bb729fb6f5b5680a4a7350234c8b290c73ec4 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 14:58:37 -0600 Subject: [PATCH 14/51] Updated documentation. --- README.md | 190 +++++++------------- examples/example_all_functionality.py | 6 + examples/example_beta_calc.py | 6 + examples/example_market_data.py | 6 + examples/example_market_maker.py | 5 + examples/example_option_risk.py | 5 + examples/example_overview_image.py | 6 + examples/example_query_and_plot.py | 5 + examples/example_read_only_functionality.py | 5 + 9 files changed, 113 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index fcfba5f8..c91c85f2 100644 --- a/README.md +++ b/README.md @@ -165,135 +165,63 @@ upper right corner. ![](https://raw.githubusercontent.com/deephaven-examples/de 7) Once [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) is launched (see [below](#launch)), accept incoming connections to [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). (May not be required for all sessions.) ![](https://raw.githubusercontent.com/deephaven-examples/deephaven-ib/main/docs/assets/allow-connections.png) +## Virtual Environment -## Launch +Interactive Brokers does not make their Python wheels available via PyPI, and the wheels are not redistributable. +As a result, installing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) requires a script to build the wheels locally before installation. +Also, to keep your development environment clean, the script creates a virtual environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). -There are multiple ways to launch [Deephaven](https://deephaven.io) with [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) -installed. The launch can either happen via a local installation or via Docker images. +### Build the Virtual Environment -For a Docker Compose example, see [docker/release/](docker/release/). - - -### (Option 1) Launch pip-installed Deephaven with Docker -- interactive - -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed in a Docker container. - -1) Create a directory for your data and scripts - ```bash - mkdir data - ``` -2) Create a directory for your Deephaven IDE configuration and notebooks - ```bash - mkdir `pwd`/.deephaven - ``` -3) Launch the system: - ```bash - # Set jvm_args to the desired JVM memory for Deephaven - docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!']); _server.start()" - ``` -4) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and logging in with the password `DeephavenRocks!`. - - -### (Option 2) Launch pip-installed Deephaven with Docker -- run a script - -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed in a Docker container. -This is a good option for production scenarios where scripts need to be run and related data needs to be visualized. - -1) Create a directory for your data and scripts +1) Install Java 17 and set the appropriate `JAVA_HOME` environment variable. +2) Check out [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) ```bash - mkdir data - # your_script.py must begin with: "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!']); _server.start()" - # Set jvm_args to the desired JVM memory for Deephaven - cp path/to/your_script.py data/your_script.py + git clone git@github.com:deephaven-examples/deephaven-ib.git ``` -2) Create a directory for your Deephaven IDE configuration and notebooks - ```bash - mkdir `pwd`/.deephaven - ``` -3) Launch the system: +3) Change to the deephaven-ib directory: ```bash - # Set jvm_args to the desired JVM memory for Deephaven - docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i /data/your_script.py + cd deephaven-ib ``` -4) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and logging in with the password `DeephavenRocks!`. - - -### (Option 3) Launch pip-installed Deephaven with a local installation (No Docker) -- interactive +4) Build a [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) virtual environment: -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed directly on your local system, without Docker. - -It is possible to use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) without docker, but this is a -new feature and has not been well tested. To do this: -1) Install `ibapi`: + To install the latest production version: ```bash - # pip installed version of ibapi is too old. You must download and install a more recent version. - export IB_VERSION=1019.04 - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" - unzip api.zip - cd ./IBJts/source/pythonclient - python3 setup.py install + python3 -m pip install -r requirements_dhib_env.txt + python3 ./dhib_env.py release ``` -2) Install Java 11 and set the appropriate `JAVA_HOME` environment variable. -3) Install [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib): + + To build the latest development version: ```bash - pip3 install --upgrade pip setuptools wheel - pip3 install deephaven-ib + python3 -m pip install -r requirements_dhib_env.txt + python3 ./dhib_env.py dev ``` -4) Launch the system: + + To see all options: ```bash - # Set jvm_args to the desired JVM memory for Deephaven - # Deephaven IDE configuration and notebooks are stored to ~/.deephaven - python3 -i -c "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]); _server.start()" + python3 ./dhib_env.py --help ``` -5) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and log in with the password `DeephavenRocks!`. -6) Use `host="localhost"` for the hostname in the examples (Windows WSL uses `host="host.docker.internal"`, since WSL is built on Docker.) +5) Take note of where the virtual environment was created. It will be in a directory like `./venv-`. + +### Activate the Virtual Environment -### (Option 4) Launch pip-installed Deephaven with a local installation (No Docker) -- run a script +To activate the virtual environment: +```bash +source ./venv-/bin/activate +``` -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed directly on your local system, without Docker. -This is a good option for production scenarios where scripts need to be run and related data needs to be visualized. +Once the virtual environment is activated, `python` and `pip` will use the virtual environment's Python and packages -- +including everything needed to run [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). -It is possible to use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) without docker, but this is a -new feature and has not been well tested. To do this: -1) Install `ibapi`: - ```bash - # pip installed version of ibapi is too old. You must download and install a more recent version. - export IB_VERSION=1016.01 - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" - unzip api.zip - cd ./IBJts/source/pythonclient - python3 setup.py install - ``` -2) Install Java 11 and set the appropriate `JAVA_HOME` environment variable. -3) Install [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib): - ```bash - pip3 install --upgrade pip setuptools wheel - pip3 install deephaven-ib - ``` -4) Launch the system and execute a custom script: - ```bash - # your_script.py must begin with: "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]); _server.start()" - # Deephaven IDE configuration and notebooks are stored to ~/.deephaven - # Set jvm_args to the desired JVM memory for Deephaven - python3 -i your_script.py - ``` -5) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and log in with the password `DeephavenRocks!`. -6) Use `host=localhost` for the hostname in the examples +### Deactivate the Virtual Environment -# Authentication +To deactivate the virtual environment: +```bash +deactivate +``` -The documentation and examples here illustrate using Deephaven's [Pre-Shared Key (PSK) authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/) -with the password `DeephavenRocks!`. Other types of Deephaven authentication can also work. -See the [Deephaven Documentation](https://deephaven.io/core/docs/) for details. +Once the virtual environment is deactivated, `python` and `pip` will use the system's Python and packages. +[deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) will not be available. -Common ways to authenticate are: -* [Anonymous Authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-anon/) -* [Pre-Shared Key (PSK) Authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/) -* [Username / Password Authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-uname-pw/) # Use deephaven-ib @@ -303,6 +231,31 @@ will depend upon how authentication is configured. In the examples here, you wi The following commands can be executed in the console. +## Start Deephaven + +To start Deephaven, run the following command in the console. This command will start Deephaven with 4GB of memory and +the password `DeephavenRocks!`. + +```python +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() +``` + +> :warning: These deephaven server commands **must** be run before importing `deephaven` or `deephaven_ib`. + +The documentation and examples here illustrate using Deephaven's [Pre-Shared Key (PSK) authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/) +with the password `DeephavenRocks!`. Other types of Deephaven authentication can also work. +See the [Deephaven Documentation](https://deephaven.io/core/docs/) for details. + +## Launch the Deephaven IDE + +To launch the Deephaven IDE, navigate to [http://localhost:10000/ide/](http://localhost:10000/ide/) in your web browser. +How you authenticate will depend upon how authentication is configured. +In the examples here, you will use the password `DeephavenRocks!`. + + ## Connect to TWS All [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) sessions need to first create a client for interacting @@ -679,7 +632,11 @@ If you can not solve your problems through either the `errors` table or through ### `Takes N positional arguments but M were given` -You may encounter an error that looks like: `Takes N positional arguments but M were given`. If you see a problem like this, your `ibapi` version does not match the version needed by [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). The [`ibapi` version in PyPI](https://pypi.org/project/ibapi/) is ancient and appears to have been abandoned by [Interactive Brokers](https://www.interactivebrokers.com/). Currently [Interactive Brokers](https://www.interactivebrokers.com/) is delivering `ibapi` either via the [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) download or via git. +You may encounter an error that looks like: `Takes N positional arguments but M were given`. +If you see a problem like this, your `ibapi` version does not match the version needed by [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). +The [`ibapi` version in PyPI](https://pypi.org/project/ibapi/) is ancient and appears to have been abandoned by [Interactive Brokers](https://www.interactivebrokers.com/). +Currently [Interactive Brokers](https://www.interactivebrokers.com/) is delivering `ibapi` either via the [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) +download or via git. To check your `ibapi` version: ```python @@ -687,17 +644,8 @@ import ibapi print(ibapi.__version__) ``` -If your version is `9.x`, it is the old version from [PyPI](https://pypi.org/project/ibapi/). To install the required `ibapi` version: -```bash -# pip installed version of ibapi is too old. You must download and install a more recent version. -export IB_VERSION=1019.04 -curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" -unzip api.zip -cd ./IBJts/source/pythonclient -python3 setup.py install -``` - -Note that the `ibapi` API is very unstable. You likely need the exact version mentioned here for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) to function. If you want a better installation experience and more flexability with the `ibapi` version, reach out to [Interactive Brokers](https://www.interactivebrokers.com/) and let them know that you want them to start publishing the latest `ibapi` versions to [PyPI](https://pypi.org). +The `ibapi` API is very unstable. If your version does not exactly match the version needed by [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), +you will need to install the correct version. Regenerate your virtual environment as described above. # Examples diff --git a/examples/example_all_functionality.py b/examples/example_all_functionality.py index 58a37b79..fda0b8f4 100644 --- a/examples/example_all_functionality.py +++ b/examples/example_all_functionality.py @@ -1,3 +1,9 @@ + +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + from typing import Dict from ibapi.contract import Contract diff --git a/examples/example_beta_calc.py b/examples/example_beta_calc.py index 41534f64..122f39fa 100644 --- a/examples/example_beta_calc.py +++ b/examples/example_beta_calc.py @@ -1,3 +1,9 @@ + +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + ## Set the API port. Default port numbers are: # 7496 - Trader Workstation, real trading # 4001 - IB Gateway, real trading diff --git a/examples/example_market_data.py b/examples/example_market_data.py index bd45faa2..84915aea 100644 --- a/examples/example_market_data.py +++ b/examples/example_market_data.py @@ -1,3 +1,9 @@ + +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + from ibapi.contract import Contract import deephaven_ib as dhib diff --git a/examples/example_market_maker.py b/examples/example_market_maker.py index 5318c897..586b21d1 100644 --- a/examples/example_market_maker.py +++ b/examples/example_market_maker.py @@ -1,4 +1,9 @@ +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + from ibapi.contract import Contract from ibapi.order import Order diff --git a/examples/example_option_risk.py b/examples/example_option_risk.py index a9513d36..11f8e3c3 100644 --- a/examples/example_option_risk.py +++ b/examples/example_option_risk.py @@ -1,5 +1,10 @@ # Compute real-time risk scenarios for a portfolio of options +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + import math from ibapi.contract import Contract from deephaven.constants import NULL_DOUBLE diff --git a/examples/example_overview_image.py b/examples/example_overview_image.py index 034a2564..99398fb7 100644 --- a/examples/example_overview_image.py +++ b/examples/example_overview_image.py @@ -1,3 +1,9 @@ + +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + import deephaven_ib as dhib print("==============================================================================================================") diff --git a/examples/example_query_and_plot.py b/examples/example_query_and_plot.py index c374b18f..9eb22568 100644 --- a/examples/example_query_and_plot.py +++ b/examples/example_query_and_plot.py @@ -1,3 +1,8 @@ +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + import deephaven_ib as dhib print("==============================================================================================================") diff --git a/examples/example_read_only_functionality.py b/examples/example_read_only_functionality.py index 423c9462..e62f024d 100644 --- a/examples/example_read_only_functionality.py +++ b/examples/example_read_only_functionality.py @@ -1,3 +1,8 @@ +import os +from deephaven_server import Server +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + from typing import Dict from ibapi.contract import Contract From 01e0ab538df55d3aa5bd9113b1e9b02b0e758633 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 30 Apr 2024 15:05:18 -0600 Subject: [PATCH 15/51] Updated documentation. --- README.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c91c85f2..cd1c3009 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,8 @@ upper right corner. ![](https://raw.githubusercontent.com/deephaven-examples/de Interactive Brokers does not make their Python wheels available via PyPI, and the wheels are not redistributable. As a result, installing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) requires a script to build the wheels locally before installation. -Also, to keep your development environment clean, the script creates a virtual environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). + +To keep your development environment clean, the script creates a virtual environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). ### Build the Virtual Environment @@ -184,13 +185,13 @@ Also, to keep your development environment clean, the script creates a virtual e ``` 4) Build a [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) virtual environment: - To install the latest production version: + To install the latest production version from PyPi: ```bash python3 -m pip install -r requirements_dhib_env.txt python3 ./dhib_env.py release ``` - To build the latest development version: + To build the latest development version from source: ```bash python3 -m pip install -r requirements_dhib_env.txt python3 ./dhib_env.py dev @@ -200,7 +201,7 @@ Also, to keep your development environment clean, the script creates a virtual e ```bash python3 ./dhib_env.py --help ``` -5) Take note of where the virtual environment was created. It will be in a directory like `./venv-`. +5) In the logs, take note of where the virtual environment was created. It will be in a directory like `./venv-`. ### Activate the Virtual Environment @@ -225,11 +226,13 @@ Once the virtual environment is deactivated, `python` and `pip` will use the sys # Use deephaven-ib -To use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), you will need to open the [Deephaven Web IDE](http://localhost:10000/ide/) -by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in your web browser. How you authenticate -will depend upon how authentication is configured. In the examples here, you will use the password `DeephavenRocks!`. +## Start Python -The following commands can be executed in the console. +To start Python with the virtual environment, run: +```bash +source ./venv-/bin/activate +python +``` ## Start Deephaven From eef03c08c9d407f5d95bb58843e8488f9642c8b6 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 09:11:00 -0600 Subject: [PATCH 16/51] Made ib version handling robust. Fixed release build. --- dhib_env.py | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/dhib_env.py b/dhib_env.py index f43e3579..06f108b3 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -16,6 +16,39 @@ IB_VERSION_DEFAULT="10.19.04" DH_VERSION_DEFAULT="0.33.3" +######################################################################################################################## +# Version Numbers +######################################################################################################################## + + +def version_tuple(version: str) -> tuple[int, ...]: + """Convert a version string to a tuple of integers. + + Args: + version: The version string to convert. + + Returns: + A tuple of integers representing the version. + """ + return tuple(map(int, (version.split(".")))) + + +def version_str(version: tuple[int, ...], wide: bool) -> str: + """Convert a version tuple to a string. + + Args: + version: The version tuple to convert. + wide: Whether to use a wide format that includes leading zeros. + + Returns: + A string representing the version. + """ + if wide: + return ".".join(f"{x:02d}" for x in version) + else: + return ".".join(map(str, version)) + + ######################################################################################################################## # Shell ######################################################################################################################## @@ -198,7 +231,8 @@ def __init__(self, version: str): def build(self) -> None: """Build the IB wheel.""" logging.warning(f"Building IB wheel: {self.version}") - shell_exec(f"cd ibwhl && IB_VERSION={self.version} docker-compose up --abort-on-container-exit") + ver_wide = version_str(version_tuple(self.version), True) + shell_exec(f"cd ibwhl && IB_VERSION={ver_wide} docker-compose up --abort-on-container-exit") def install(self, v: Venv) -> None: """Install the IB wheel into a virtual environment. @@ -207,9 +241,8 @@ def install(self, v: Venv) -> None: v: The virtual environment to install the wheel into. """ logging.warning(f"Installing IB wheel in venv: {self.version} {v.path}") - # Strip the 0 from the version number - mod_ver = self.version.replace(".0", ".") - v.pip_install(Path(f"ibwhl/dist/ibapi-{mod_ver}-py3-none-any.whl").absolute()) + ver_narrow = version_str(version_tuple(self.version), False) + v.pip_install(Path(f"ibwhl/dist/ibapi-{ver_narrow}-py3-none-any.whl").absolute()) ######################################################################################################################## @@ -310,10 +343,10 @@ def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): """Create a release environment.""" logging.warning(f"Creating release environment: python={python} dh_ib_version={dh_ib_version}") - wheel = download_wheel(python, "deephaven-ib", dh_ib_version) + wheel = download_wheel(python, "deephaven_ib", dh_ib_version) deps = pkg_dependencies(wheel) - ib_version = deps["ibapi"] - dh_version = deps["deephaven-server"] + ib_version = deps["ibapi"].replace("==", "") + dh_version = deps["deephaven-server"].replace("==", "") v = Venv(True, python, dh_version, ib_version, dh_ib_version, delete_venv) From 122a7c7e8399a6747f1bd082120a60a9f8223f2b Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 10:19:23 -0600 Subject: [PATCH 17/51] Removed all docker requirements. Replaced IB docker build with local functionality in the script. --- README.md | 12 ++++---- dhib_env.py | 61 ++++++++++++++++++++++++++++++++++------ ibwhl/README.md | 30 -------------------- ibwhl/build_ibpy.sh | 39 ------------------------- ibwhl/docker-compose.yml | 29 ------------------- 5 files changed, 57 insertions(+), 114 deletions(-) delete mode 100644 ibwhl/README.md delete mode 100755 ibwhl/build_ibpy.sh delete mode 100644 ibwhl/docker-compose.yml diff --git a/README.md b/README.md index cd1c3009..f64de3f0 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,6 @@ You may want to combine data from other sources with your IB data. [Deephaven]( * [Kafka](https://deephaven.io/core/docs/how-to-guides/kafka-topics/). See the [Deephaven Documentation](https://deephaven.io/core/docs) for details. -Files placed in the `./docker/data/` directory are visible in the Docker container at `/data/`. -See [Access your file system with Docker data volumes](https://deephaven.io/core/docs/conceptual/docker-data-volumes/) for details. - # Run deephaven-ib Follow these steps to run a [Deephaven](https://deephaven.io) plus [Interactive Brokers](https://interactivebrokers.com) system. @@ -264,7 +261,8 @@ In the examples here, you will use the password `DeephavenRocks!`. All [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) sessions need to first create a client for interacting with [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). -`host` is the computer to connect to. When using [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) inside +`host` is the computer to connect to. When using [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) locally, `host` is usually set to `localhost`. +When using [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) inside of Docker, `host` should be set to `host.docker.internal`. `port` is the network port [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) @@ -288,7 +286,7 @@ For a read-write session that allows trading: ```python import deephaven_ib as dhib -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, read_only=False) +client = dhib.IbSessionTws(host="localhost", port=7497, read_only=False) client.connect() ``` @@ -296,7 +294,7 @@ For a read-only session that does not allow trading: ```python import deephaven_ib as dhib -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, read_only=True) +client = dhib.IbSessionTws(host="localhost", port=7497, read_only=True) client.connect() ``` @@ -304,7 +302,7 @@ For a read-only financial advisor (FA) session that does not allow trading: ```python import deephaven_ib as dhib -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, read_only=True, is_fa=True) +client = dhib.IbSessionTws(host="localhost", port=7497, read_only=True, is_fa=True) client.connect() ``` diff --git a/dhib_env.py b/dhib_env.py index 06f108b3..2e6825cf 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -67,6 +67,26 @@ def shell_exec(cmd: str) -> None: raise Exception(f"Error executing shell command: {cmd}") +######################################################################################################################## +# URL +######################################################################################################################## + + +def url_download(url: str, path: Union[str, Path]) -> None: + """Download a file from a URL. + + Args: + url: The URL to download from. + path: The path to save the downloaded file to. + """ + logging.warning(f"Downloading file: {url}, path: {path}") + response = requests.get(url) + response.raise_for_status() + + with open(path, "wb") as f: + f.write(response.content) + + ######################################################################################################################## # Package Query Functions ######################################################################################################################## @@ -226,13 +246,36 @@ def __init__(self, version: str): Args: version: The version of the IB wheel. """ - self.version = version + self.version = version_tuple(version) - def build(self) -> None: - """Build the IB wheel.""" + def build(self, v: Venv) -> None: + """Build the IB wheel. + + Interactive Brokers does not make their Python wheels available via PyPI, + and the wheels are not redistributable. + As a result, we need to build the IB wheel locally. + + Args: + v: The virtual environment to build the wheel in. + """ logging.warning(f"Building IB wheel: {self.version}") - ver_wide = version_str(version_tuple(self.version), True) - shell_exec(f"cd ibwhl && IB_VERSION={ver_wide} docker-compose up --abort-on-container-exit") + + shutil.rmtree("build/ib", ignore_errors=True) + shutil.rmtree("dist/ib", ignore_errors=True) + + os.makedirs("build/ib", exist_ok=True) + os.makedirs("dist/ib", exist_ok=True) + + logging.warning(f"Downloading IB API version {self.version}") + ver_ib = f"{self.version[0]:02d}{self.version[1]:02d}.{self.version[2]:02d}" + url_download(f"https://interactivebrokers.github.io/downloads/twsapi_macunix.{ver_ib}.zip", "build/ib/api.zip") + + logging.warning(f"Unzipping IB API") + shell_exec("cd build/ib && unzip api.zip") + + logging.warning(f"Building IB Python API") + shell_exec(f"cd build/ib/IBJts/source/pythonclient && {v.python} -m build --wheel") + shell_exec("cp build/ib/IBJts/source/pythonclient/dist/* dist/ib/") def install(self, v: Venv) -> None: """Install the IB wheel into a virtual environment. @@ -241,8 +284,8 @@ def install(self, v: Venv) -> None: v: The virtual environment to install the wheel into. """ logging.warning(f"Installing IB wheel in venv: {self.version} {v.path}") - ver_narrow = version_str(version_tuple(self.version), False) - v.pip_install(Path(f"ibwhl/dist/ibapi-{ver_narrow}-py3-none-any.whl").absolute()) + ver_narrow = version_str(self.version, False) + v.pip_install(Path(f"dist/ib/ibapi-{ver_narrow}-py3-none-any.whl").absolute()) ######################################################################################################################## @@ -317,7 +360,7 @@ def dev(python: str, dh_version: str, ib_version: str, dh_ib_version: Optional[s v = Venv(False, python, dh_version, ib_version, dh_ib_version, delete_venv) ib_wheel = IbWheel(ib_version) - ib_wheel.build() + ib_wheel.build(v) ib_wheel.install(v) v.pip_install("deephaven-server", dh_version) @@ -351,7 +394,7 @@ def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): v = Venv(True, python, dh_version, ib_version, dh_ib_version, delete_venv) ib_wheel = IbWheel(ib_version) - ib_wheel.build() + ib_wheel.build(v) ib_wheel.install(v) logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") diff --git a/ibwhl/README.md b/ibwhl/README.md deleted file mode 100644 index 0d89433c..00000000 --- a/ibwhl/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Interactive Brokers Python Wheels Builder - -This directory provides a Docker Compose setup for building Python wheels for the Interactive Brokers API. - -## Requirements - -- Docker -- Docker Compose - -## Usage - -1. Set the `IB_VERSION` environment variable to the version of Interactive Brokers' API you want to use. -2. Run `docker compose up` from the directory containing the Docker Compose file. This will build the wheels. -3. After the build process is complete, the wheels will be available in the `./dist` directory. -4. To install the wheels, run `pip install ./dist/*.whl`. - -> **Note:** Steps 1 and 2 can be combined as `IB_VERSION=10.19.04 docker compose up`. - -## Details - -The build process is defined in the `build_ibpy.sh` script. -The `./build` directory contains the scratch work for building the wheels. - -By using Docker, you can build the wheels in a clean environment that is isolated from your local machine. -This setup will work on any platform that supports Docker. - -## Note - -Interactive Brokers does not make their Python wheels available via PyPI, and the wheels are not redistributable. -This script lets you build the wheels locally, and then install them via pip. \ No newline at end of file diff --git a/ibwhl/build_ibpy.sh b/ibwhl/build_ibpy.sh deleted file mode 100755 index 36a791f5..00000000 --- a/ibwhl/build_ibpy.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# This script is used to build Python wheels for the Interactive Brokers' API. - -# Exit script on first error -set -e - -# check that IB_VERSION is set -if [ -z "$IB_VERSION" ]; then - echo "IB_VERSION must be set" - exit 1 -fi - -# IB has a funky versioning scheme, so we need to strip out the periods in some places before download -IB_VERSION_DOWNLOAD=$(echo ${IB_VERSION} | sed 's/[.]//') - -if [ ! -d "./build" ]; then - mkdir ./build -fi - -if [ ! -d "./dist" ]; then - mkdir ./dist -fi - -rm -rf ./build/* -rm -rf ./dist/* - -pushd ./build - -python3 -m pip install build - -echo "Downloading IB API version ${IB_VERSION} (${IB_VERSION_DOWNLOAD})" -curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION_DOWNLOAD}.zip" -unzip api.zip -cd ./IBJts/source/pythonclient -python3 -m build --wheel -popd -cp ./build/IBJts/source/pythonclient/dist/* ./dist/ - diff --git a/ibwhl/docker-compose.yml b/ibwhl/docker-compose.yml deleted file mode 100644 index d37a910f..00000000 --- a/ibwhl/docker-compose.yml +++ /dev/null @@ -1,29 +0,0 @@ - -# This is a Docker Compose file for building Interactive Brokers python wheels. -# Ideally, the wheels would be published by IB for installation via pip, but -# (1) IB does not make them available via pypi, and -# (2) the wheels are not redistributable. -# This script lets you build the wheels locally, and then install them via pip. -# -# To build the wheels, run `docker-compose up` from the directory containing this file. -# To install the wheels, run `pip install ./dist/*.whl`. -# After running the script, the wheels will be available in the `./dist` directory. -# The `./build` directory contains the scratch work for building the wheels. -# -# The build process is defined in the `build_ibpy.sh` script. -# -# By using Docker, you can build the wheels in a clean environment that is isolated from your local machine. -# It also will work on any platform that supports Docker. - -version: '3.8' - -services: - build: - image: python:3.10 - environment: - - IB_VERSION - volumes: - - .:/app - - ./build:/build - - ./dist:/dist - command: /bin/bash -c "/app/build_ibpy.sh" \ No newline at end of file From 70c5cb0982d8a11eade25712b07265cd118cb0ab Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 10:32:13 -0600 Subject: [PATCH 18/51] Release the hard version constraint on deephaven. --- .github/workflows/build-and-publish.yml | 2 +- dhib_env.py | 12 +++++++++--- setup.py | 18 +++++++++++++----- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index c335ecfb..5128e006 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -13,7 +13,7 @@ on: env: IB_VERSION: 10.19.04 - DH_VERSION: 0.33.3 + DH_VERSION: 0.33 jobs: build-ib-whl: diff --git a/dhib_env.py b/dhib_env.py index 2e6825cf..b13fc8fd 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -14,7 +14,7 @@ import requests IB_VERSION_DEFAULT="10.19.04" -DH_VERSION_DEFAULT="0.33.3" +DH_VERSION_DEFAULT="0.33" ######################################################################################################################## # Version Numbers @@ -230,7 +230,13 @@ def pip_install(self, package: Union[str, Path], version: Optional[str] = None) if isinstance(package, Path): package = package.absolute() - ver = f"=={version}" if version else "" + if not version: + ver = "" + elif len(version.split(".")) > 2: + ver = f"=={version}" + else: + ver = f"~={version}" + cmd = f"""{self.python} -m pip install {package}{ver}""" shell_exec(cmd) @@ -389,7 +395,7 @@ def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): wheel = download_wheel(python, "deephaven_ib", dh_ib_version) deps = pkg_dependencies(wheel) ib_version = deps["ibapi"].replace("==", "") - dh_version = deps["deephaven-server"].replace("==", "") + dh_version = deps["deephaven-server"].replace("==", "").replace("~=", "") v = Venv(True, python, dh_version, ib_version, dh_ib_version, delete_venv) diff --git a/setup.py b/setup.py index 21f345ea..58cdce92 100644 --- a/setup.py +++ b/setup.py @@ -8,17 +8,25 @@ dh_ib_version = os.getenv("DH_IB_VERSION") if not dh_ib_version: - raise Exception("deephaven-ib version must be set via the DH_IB_VERSION environment varialble.") + raise Exception("deephaven-ib version must be set via the DH_IB_VERSION environment variable.") dh_version = os.getenv("DH_VERSION") if not dh_version: - raise Exception("deephaven version must be set via the DH_VERSION environment varialble.") + raise Exception("deephaven version must be set via the DH_VERSION environment variable.") ib_version = os.getenv("IB_VERSION") if not ib_version: - raise Exception("ibapi version must be set via the IB_VERSION environment varialble.") + raise Exception("ibapi version must be set via the IB_VERSION environment variable.") + + +def add_version_constraint(package_name, version): + if len(dh_version.split(".")) < 3: + return f"deephaven-server~={dh_version}" + else: + return f"{package_name}=={version}" + setuptools.setup( name="deephaven_ib", @@ -49,9 +57,9 @@ packages=setuptools.find_packages(where="src"), python_requires=">=3.6", install_requires=[ - f"deephaven-server=={dh_version}", + add_version_constraint("deephaven-server", dh_version), "pandas", - f"ibapi=={ib_version}", + add_version_constraint("ibapi", ib_version), "lxml", "ratelimit", ], From 0b51312b5215d173c3d41f3e0416d73ade3f1964 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 10:46:21 -0600 Subject: [PATCH 19/51] Updated the minimum python version to avoid core dump in jpy. https://github.com/deephaven/deephaven-core/issues/5440 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 58cdce92..393eac28 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def add_version_constraint(package_name, version): ], package_dir={"": "src"}, packages=setuptools.find_packages(where="src"), - python_requires=">=3.6", + python_requires=">=3.10", install_requires=[ add_version_constraint("deephaven-server", dh_version), "pandas", From b44eab1436d06ecd99cb3b13c58120e9a683ac48 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 13:12:30 -0600 Subject: [PATCH 20/51] Fix broken dependency version handling. --- .github/workflows/build-and-publish.yml | 2 +- dhib_env.py | 71 +++++++++++++++++++------ setup.py | 34 +++++++++--- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 5128e006..bc527344 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -13,7 +13,7 @@ on: env: IB_VERSION: 10.19.04 - DH_VERSION: 0.33 + DH_VERSION: 0.33.0 jobs: build-ib-whl: diff --git a/dhib_env.py b/dhib_env.py index b13fc8fd..a61f8f10 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -5,6 +5,7 @@ import atexit import logging import os +import re import shutil from pathlib import Path from types import ModuleType @@ -14,7 +15,7 @@ import requests IB_VERSION_DEFAULT="10.19.04" -DH_VERSION_DEFAULT="0.33" +DH_VERSION_DEFAULT="0.33.0" ######################################################################################################################## # Version Numbers @@ -49,6 +50,27 @@ def version_str(version: tuple[int, ...], wide: bool) -> str: return ".".join(map(str, version)) +def version_assert_format(version: str) -> None: + """Assert that a version string is formatted correctly. + + Args: + version: The version string to check. + + Raises: + ValueError: If the version string is not formatted correctly. + """ + if not version: + raise ValueError("Version string is empty.") + + # check if the version string is in semver format + pattern1 = re.compile(r"^([0-9]\d*)\.([0-9]\d*)\.([0-9]\d*)$") + pattern2 = re.compile(r"^([0-9]\d*)\.([0-9]\d*)\.([0-9]\d*)\.dev([0-9]\d*)$") + is_semver = bool(pattern1.match(version)) or bool(pattern2.match(version)) + + if not is_semver: + raise ValueError(f"Version string is not in semver format: {version}") + + ######################################################################################################################## # Shell ######################################################################################################################## @@ -218,26 +240,20 @@ def python(self) -> str: """The path to the Python executable in the virtual environment.""" return os.path.join(self.path, "bin", "python") - def pip_install(self, package: Union[str, Path], version: Optional[str] = None) -> None: + def pip_install(self, package: Union[str, Path], version: str = "") -> None: """Install a package into the virtual environment. Args: package: The name of the package to install. - version: The version of the package to install. If None, the latest version will be installed. + version: The version constraint of the package to install. If None, the latest version will be installed. + For example, provide "==1.2.3" to install version 1.2.3. """ logging.warning(f"Installing package in venv: {package}, version: {version}, venv: {self.path}") if isinstance(package, Path): package = package.absolute() - if not version: - ver = "" - elif len(version.split(".")) > 2: - ver = f"=={version}" - else: - ver = f"~={version}" - - cmd = f"""{self.python} -m pip install {package}{ver}""" + cmd = f"""{self.python} -m pip install {package}{version}""" shell_exec(cmd) @@ -351,25 +367,39 @@ def cli(): @click.command() @click.option('--python', default="python3", help='The path to the Python executable to use.') @click.option('--dh_version', default=DH_VERSION_DEFAULT, help='The version of Deephaven.') +@click.option('--dh_version_exact', default=None, help='The exact version of Deephaven.') @click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') @click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') @click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') -def dev(python: str, dh_version: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool): +def dev(python: str, dh_version: str, dh_version_exact: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool): """Create a development environment.""" - logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}") + logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, dh_version_exact={dh_version_exact}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}") + + if dh_version_exact: + if dh_version != DH_VERSION_DEFAULT: + raise ValueError(f"Cannot specify both dh_version={dh_version} and dh_version_exact={dh_version_exact}") + + dh_version = dh_version_exact + dh_version_pip = f"=={dh_version}" + else: + dh_version_pip = f"~={dh_version}" use_dev = dh_ib_version is None if dh_ib_version is None: dh_ib_version = "0.0.0.dev0" + version_assert_format(dh_version) + version_assert_format(ib_version) + version_assert_format(dh_ib_version) + v = Venv(False, python, dh_version, ib_version, dh_ib_version, delete_venv) ib_wheel = IbWheel(ib_version) ib_wheel.build(v) ib_wheel.install(v) - v.pip_install("deephaven-server", dh_version) + v.pip_install("deephaven-server", dh_version_pip) if use_dev: logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") @@ -379,7 +409,7 @@ def dev(python: str, dh_version: str, ib_version: str, dh_ib_version: Optional[s else: logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") logging.warning(f"*** INSTALLED deephaven-ib MAY BE INCONSISTENT WITH INSTALLED DEPENDENCIES ***") - v.pip_install("deephaven-ib", dh_ib_version) + v.pip_install("deephaven-ib", f"=={dh_ib_version}") success(v) @@ -397,6 +427,15 @@ def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): ib_version = deps["ibapi"].replace("==", "") dh_version = deps["deephaven-server"].replace("==", "").replace("~=", "") + version_assert_format(dh_version) + version_assert_format(ib_version) + + if dh_ib_version: + version_assert_format(dh_ib_version) + dh_ib_version_pip = f"=={dh_ib_version}" + else: + dh_ib_version_pip = "" + v = Venv(True, python, dh_version, ib_version, dh_ib_version, delete_venv) ib_wheel = IbWheel(ib_version) @@ -404,7 +443,7 @@ def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): ib_wheel.install(v) logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") - v.pip_install("deephaven-ib", dh_ib_version) + v.pip_install("deephaven-ib", dh_ib_version_pip) success(v) diff --git a/setup.py b/setup.py index 393eac28..3976cc4f 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import os +import re import setuptools @@ -21,12 +22,31 @@ raise Exception("ibapi version must be set via the IB_VERSION environment variable.") -def add_version_constraint(package_name, version): - if len(dh_version.split(".")) < 3: - return f"deephaven-server~={dh_version}" - else: - return f"{package_name}=={version}" +def version_assert_format(version: str) -> None: + """Assert that a version string is formatted correctly. + Args: + version: The version string to check. + + Raises: + ValueError: If the version string is not formatted correctly. + """ + if not version: + raise ValueError("Version string is empty.") + + # check if the version string is in semver format + # check if the version string is in semver format + pattern1 = re.compile(r"^([0-9]\d*)\.([0-9]\d*)\.([0-9]\d*)$") + pattern2 = re.compile(r"^([0-9]\d*)\.([0-9]\d*)\.([0-9]\d*)\.dev([0-9]\d*)$") + is_semver = bool(pattern1.match(version)) or bool(pattern2.match(version)) + + if not is_semver: + raise ValueError(f"Version string is not in semver format: {version}") + + +version_assert_format(dh_ib_version) +version_assert_format(dh_version) +version_assert_format(ib_version) setuptools.setup( name="deephaven_ib", @@ -57,9 +77,9 @@ def add_version_constraint(package_name, version): packages=setuptools.find_packages(where="src"), python_requires=">=3.10", install_requires=[ - add_version_constraint("deephaven-server", dh_version), + f"deephaven-server~={dh_version}", "pandas", - add_version_constraint("ibapi", ib_version), + f"ibapi=={ib_version}", "lxml", "ratelimit", ], From 7aa5b8c4028ecb3c760e54f12a6256bb2fa43db1 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 14:01:30 -0600 Subject: [PATCH 21/51] Fix broken short rate downloads. --- src/deephaven_ib/_internal/short_rates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/deephaven_ib/_internal/short_rates.py b/src/deephaven_ib/_internal/short_rates.py index ede01ec7..7252fc5b 100644 --- a/src/deephaven_ib/_internal/short_rates.py +++ b/src/deephaven_ib/_internal/short_rates.py @@ -71,7 +71,8 @@ def write(self, line: str) -> None: def load_short_rates() -> Table: """Downloads the short rates from the IB FTP site and returns them as a table.""" - host: str = "ftp3.interactivebrokers.com" + # See: https://www.ibkrguides.com/kb/article-2024.htm + host: str = "ftp2.interactivebrokers.com" user: str = "shortstock" with ftplib.FTP(host=host, user=user) as ftp, IBFtpWriter() as p: From 2905ca65cd87e78e9be2dfb1409654defefe0c1a Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 14:02:22 -0600 Subject: [PATCH 22/51] Fix DH queries that are broken. Better error messages. --- src/deephaven_ib/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/deephaven_ib/__init__.py b/src/deephaven_ib/__init__.py index a26af40b..c0bd83fb 100644 --- a/src/deephaven_ib/__init__.py +++ b/src/deephaven_ib/__init__.py @@ -550,7 +550,7 @@ def annotate_ticks(t): return rst - def deephaven_ib_float_value(s: str) -> Optional[float]: + def deephaven_ib_float_value(s: Optional[str]) -> Optional[float]: if not s: return NULL_DOUBLE @@ -638,7 +638,7 @@ def deephaven_ib_parse_note(note:str, key:str) -> Optional[str]: "orders_status": tables_raw["raw_orders_status"] \ .last_by("PermId") \ .move_columns_up(["ReceiveTime", "PermId", "ClientId", "OrderId", "ParentId"]), - "bars_historical": annotate_ticks(tables_raw["raw_bars_historical"]).last_by(["Request", "Timestamp", "ContractId"]), + "bars_historical": annotate_ticks(tables_raw["raw_bars_historical"]).last_by(["RequestId", "Timestamp", "ContractId"]), "bars_realtime": annotate_ticks(tables_raw["raw_bars_realtime"]), "ticks_efp": annotate_ticks(tables_raw["raw_ticks_efp"]), "ticks_generic": annotate_ticks(tables_raw["raw_ticks_generic"]), @@ -692,8 +692,12 @@ def get_registered_contract(self, contract: Contract) -> RegisteredContract: """ self._assert_connected() - cd = self._client.contract_registry.request_contract_details_blocking(contract) - return RegisteredContract(query_contract=contract, contract_details=cd) + + try: + cd = self._client.contract_registry.request_contract_details_blocking(contract) + return RegisteredContract(query_contract=contract, contract_details=cd) + except Exception as e: + raise Exception(f"Error getting registered contract: contract={contract} {e}") def request_contracts_matching(self, pattern: str) -> Request: """Request contracts matching a pattern. Results are returned in the ``contracts_matching`` table. From 26fe2d5c6fb3d61a8511156a08ae3f4d04e97a69 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 14:13:52 -0600 Subject: [PATCH 23/51] Fix DH examples. Add a script to start and run deephaven. --- README.md | 14 +++++++++-- examples/example_all_functionality.py | 25 +++++++++----------- examples/example_beta_calc.py | 7 ++---- examples/example_market_data.py | 7 ++---- examples/example_market_maker.py | 7 ++---- examples/example_option_risk.py | 7 ++---- examples/example_overview_image.py | 7 ++---- examples/example_query_and_plot.py | 8 +++---- examples/example_read_only_functionality.py | 26 ++++++++++----------- examples/run_deephaven.py | 16 +++++++++++++ 10 files changed, 64 insertions(+), 60 deletions(-) create mode 100644 examples/run_deephaven.py diff --git a/README.md b/README.md index f64de3f0..74fd264a 100644 --- a/README.md +++ b/README.md @@ -233,14 +233,19 @@ python ## Start Deephaven -To start Deephaven, run the following command in the console. This command will start Deephaven with 4GB of memory and -the password `DeephavenRocks!`. +To start Deephaven, run the following python script. It can be found at [./examples/run_deephaven.py](./examples/run_deephaven.py). +This command will start Deephaven with 4GB of memory and the password `DeephavenRocks!`. ```python import os +from time import sleep from deephaven_server import Server + _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) _server.start() + +while True: + sleep(1) ``` > :warning: These deephaven server commands **must** be run before importing `deephaven` or `deephaven_ib`. @@ -251,6 +256,11 @@ See the [Deephaven Documentation](https://deephaven.io/core/docs/) for details. ## Launch the Deephaven IDE +Once the Deephaven server is started, you can launch the Deephaven IDE. +The Deephaven IDE is a web-based interface for working with Deephaven. +Once in the IDE, you can run queries, create notebooks, and visualize data. +You can also run all of the example code below and the more complex examples in [./examples](./examples). + To launch the Deephaven IDE, navigate to [http://localhost:10000/ide/](http://localhost:10000/ide/) in your web browser. How you authenticate will depend upon how authentication is configured. In the examples here, you will use the password `DeephavenRocks!`. diff --git a/examples/example_all_functionality.py b/examples/example_all_functionality.py index fda0b8f4..2ff8783d 100644 --- a/examples/example_all_functionality.py +++ b/examples/example_all_functionality.py @@ -1,8 +1,5 @@ -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() +# Run this example in the Deephaven IDE Console from typing import Dict @@ -20,7 +17,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, client_id=0, download_short_rates=True, read_only=False) +client = dhib.IbSessionTws(host="localhost", port=7497, client_id=0, download_short_rates=True, read_only=False) print(f"IsConnected: {client.is_connected()}") client.connect() @@ -135,8 +132,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "20240119" - contract.strike = 138.5 + contract.lastTradeDateOrContractMonth = "20260116" + contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" rst["option_1"] = contract @@ -169,8 +166,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "FOP" contract.exchange = "CME" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "202312" - contract.strike = 4700 + contract.lastTradeDateOrContractMonth = "202409" + contract.strike = 5000 contract.right = "C" contract.multiplier = "50" rst["futureoption_1"] = contract @@ -179,14 +176,14 @@ def get_contracts() -> Dict[str, Contract]: contract = Contract() # enter CUSIP as symbol - contract.symbol = "912828C57" + contract.symbol = "084664BL4" contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" rst["bond_1"] = contract contract = Contract() - contract.conId = 147554578 + contract.conId = 577489715 contract.exchange = "SMART" rst["bond_2"] = contract @@ -322,7 +319,7 @@ def get_contracts() -> Dict[str, Contract]: # enter CUSIP as symbol contract = Contract() -contract.symbol = "IBCID411964960" +contract.conId = 505885457 contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" @@ -462,8 +459,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" -contract.lastTradeDateOrContractMonth = "20240119" -contract.strike = 138.5 +contract.lastTradeDateOrContractMonth = "20260116" +contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" diff --git a/examples/example_beta_calc.py b/examples/example_beta_calc.py index 122f39fa..2c14cc7f 100644 --- a/examples/example_beta_calc.py +++ b/examples/example_beta_calc.py @@ -1,8 +1,5 @@ -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() +# Run this example in the Deephaven IDE Console ## Set the API port. Default port numbers are: # 7496 - Trader Workstation, real trading @@ -19,7 +16,7 @@ else: read_only_api = True -client = dhib.IbSessionTws(host="host.docker.internal", port=API_PORT, read_only=read_only_api) +client = dhib.IbSessionTws(host="localhost", port=API_PORT, read_only=read_only_api) client.connect() if client.is_connected(): diff --git a/examples/example_market_data.py b/examples/example_market_data.py index 84915aea..c5df7499 100644 --- a/examples/example_market_data.py +++ b/examples/example_market_data.py @@ -1,8 +1,5 @@ -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() +# Run this example in the Deephaven IDE Console from ibapi.contract import Contract @@ -12,7 +9,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, download_short_rates=False) +client = dhib.IbSessionTws(host="localhost", port=7497, download_short_rates=False) client.connect() # Makes all tables global variables so that they are displayed in the user interface diff --git a/examples/example_market_maker.py b/examples/example_market_maker.py index 586b21d1..f4424676 100644 --- a/examples/example_market_maker.py +++ b/examples/example_market_maker.py @@ -1,8 +1,5 @@ -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() +# Run this example in the Deephaven IDE Console from ibapi.contract import Contract from ibapi.order import Order @@ -24,7 +21,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, client_id=0, download_short_rates=False, read_only=False) +client = dhib.IbSessionTws(host="localhost", port=7497, client_id=0, download_short_rates=False, read_only=False) print(f"IsConnected: {client.is_connected()}") client.connect() diff --git a/examples/example_option_risk.py b/examples/example_option_risk.py index 11f8e3c3..3c4bf0ce 100644 --- a/examples/example_option_risk.py +++ b/examples/example_option_risk.py @@ -1,9 +1,6 @@ # Compute real-time risk scenarios for a portfolio of options -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() +# Run this example in the Deephaven IDE Console import math from ibapi.contract import Contract @@ -16,7 +13,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, download_short_rates=False) +client = dhib.IbSessionTws(host="localhost", port=7497, download_short_rates=False) client.connect() print("==============================================================================================================") diff --git a/examples/example_overview_image.py b/examples/example_overview_image.py index 99398fb7..c705d3a5 100644 --- a/examples/example_overview_image.py +++ b/examples/example_overview_image.py @@ -1,8 +1,5 @@ -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() +# Run this example in the Deephaven IDE Console import deephaven_ib as dhib @@ -10,7 +7,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497) +client = dhib.IbSessionTws(host="localhost", port=7497) client.connect() from ibapi.contract import Contract diff --git a/examples/example_query_and_plot.py b/examples/example_query_and_plot.py index 9eb22568..82f29e7f 100644 --- a/examples/example_query_and_plot.py +++ b/examples/example_query_and_plot.py @@ -1,7 +1,5 @@ -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() + +# Run this example in the Deephaven IDE Console import deephaven_ib as dhib @@ -9,7 +7,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497) +client = dhib.IbSessionTws(host="localhost", port=7497) client.connect() from ibapi.contract import Contract diff --git a/examples/example_read_only_functionality.py b/examples/example_read_only_functionality.py index e62f024d..a8ef9144 100644 --- a/examples/example_read_only_functionality.py +++ b/examples/example_read_only_functionality.py @@ -1,7 +1,5 @@ -import os -from deephaven_server import Server -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() + +# Run this example in the Deephaven IDE Console from typing import Dict @@ -15,7 +13,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, client_id=0, download_short_rates=True, read_only=True) +client = dhib.IbSessionTws(host="localhost", port=7497, client_id=0, download_short_rates=True, read_only=True) print(f"IsConnected: {client.is_connected()}") client.connect() @@ -130,8 +128,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "20240119" - contract.strike = 138.5 + contract.lastTradeDateOrContractMonth = "20260116" + contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" rst["option_1"] = contract @@ -164,8 +162,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "FOP" contract.exchange = "CME" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "202312" - contract.strike = 4700 + contract.lastTradeDateOrContractMonth = "202409" + contract.strike = 5000 contract.right = "C" contract.multiplier = "50" rst["futureoption_1"] = contract @@ -174,14 +172,14 @@ def get_contracts() -> Dict[str, Contract]: contract = Contract() # enter CUSIP as symbol - contract.symbol = "912828C57" + contract.symbol = "084664BL4" contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" rst["bond_1"] = contract contract = Contract() - contract.conId = 147554578 + contract.conId = 577489715 contract.exchange = "SMART" rst["bond_2"] = contract @@ -317,7 +315,7 @@ def get_contracts() -> Dict[str, Contract]: # enter CUSIP as symbol contract = Contract() -contract.symbol = "IBCID411964960" +contract.conId = 505885457 contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" @@ -457,8 +455,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" -contract.lastTradeDateOrContractMonth = "20240119" -contract.strike = 138.5 +contract.lastTradeDateOrContractMonth = "20260116" +contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" diff --git a/examples/run_deephaven.py b/examples/run_deephaven.py new file mode 100644 index 00000000..3aac6d7d --- /dev/null +++ b/examples/run_deephaven.py @@ -0,0 +1,16 @@ + +# This script runs the Deephaven server with the specified port and JVM arguments. +# The server will run until the script is interrupted. +# +# To connect to the Deephaven IDE, navigate to https://localhost:10000 +# The login password is: DeephavenRocks! + +import os +from time import sleep +from deephaven_server import Server + +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + +while True: + sleep(1) From 018d8f0fb1ec394dee34467003a5908070ca5b34 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 1 May 2024 14:28:17 -0600 Subject: [PATCH 24/51] Add an option to create a venv for use in pycharm. --- README.md | 21 +++++++++++++++------ dhib_env.py | 22 ++++++++++++---------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 74fd264a..0e82ef35 100644 --- a/README.md +++ b/README.md @@ -182,22 +182,31 @@ To keep your development environment clean, the script creates a virtual environ ``` 4) Build a [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) virtual environment: - To install the latest production version from PyPi: + First, install the dependencies needed to run the script: ```bash python3 -m pip install -r requirements_dhib_env.txt + ``` + + To see all options: + ```bash + python3 ./dhib_env.py --help + ``` + + To install the latest production version from PyPi: + ```bash python3 ./dhib_env.py release ``` - To build the latest development version from source: + To install the latest development version from source: ```bash - python3 -m pip install -r requirements_dhib_env.txt python3 ./dhib_env.py dev ``` - - To see all options: + + To create a venv for developing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) in PyCharm: (This will not install deephaven-ib.) ```bash - python3 ./dhib_env.py --help + python3 ./dhib_env.py dev --install_dhib false ``` + 5) In the logs, take note of where the virtual environment was created. It will be in a directory like `./venv-`. ### Activate the Virtual Environment diff --git a/dhib_env.py b/dhib_env.py index a61f8f10..ae4a4069 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -371,7 +371,8 @@ def cli(): @click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') @click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') @click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') -def dev(python: str, dh_version: str, dh_version_exact: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool): +@click.option('--install_dhib', default=True, help='Whether to install deephaven-ib. If set to false, the resulting venv can be used to develop deephaven-ib in PyCharm or other development environments.') +def dev(python: str, dh_version: str, dh_version_exact: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool, install_dhib: bool): """Create a development environment.""" logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, dh_version_exact={dh_version_exact}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}") @@ -401,15 +402,16 @@ def dev(python: str, dh_version: str, dh_version_exact: str, ib_version: str, dh v.pip_install("deephaven-server", dh_version_pip) - if use_dev: - logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") - dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version) - dh_ib_wheel.build(v) - dh_ib_wheel.install(v) - else: - logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") - logging.warning(f"*** INSTALLED deephaven-ib MAY BE INCONSISTENT WITH INSTALLED DEPENDENCIES ***") - v.pip_install("deephaven-ib", f"=={dh_ib_version}") + if install_dhib: + if use_dev: + logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") + dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version) + dh_ib_wheel.build(v) + dh_ib_wheel.install(v) + else: + logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") + logging.warning(f"*** INSTALLED deephaven-ib MAY BE INCONSISTENT WITH INSTALLED DEPENDENCIES ***") + v.pip_install("deephaven-ib", f"=={dh_ib_version}") success(v) From fc54e17dbd353da9af6aa6d46512bd84b20ddfd0 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 13:31:01 -0600 Subject: [PATCH 25/51] Upgrade to Deephaven 0.34.1 --- .github/workflows/Dockerfile.server | 2 +- .github/workflows/build-and-publish.yml | 2 +- dhib_env.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Dockerfile.server b/.github/workflows/Dockerfile.server index 24ed1d85..9f3d288f 100644 --- a/.github/workflows/Dockerfile.server +++ b/.github/workflows/Dockerfile.server @@ -1,6 +1,6 @@ ARG DH_VERSION -FROM ghcr.io/deephaven/server:${DH_VERSION} +FROM ghcr.io/deephaven/server-ui:${DH_VERSION} RUN apt update && \ apt install -y python3-pip python3-venv curl zip && \ diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index bc527344..99ce8218 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -13,7 +13,7 @@ on: env: IB_VERSION: 10.19.04 - DH_VERSION: 0.33.0 + DH_VERSION: 0.34.1 jobs: build-ib-whl: diff --git a/dhib_env.py b/dhib_env.py index ae4a4069..1c176831 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -15,7 +15,7 @@ import requests IB_VERSION_DEFAULT="10.19.04" -DH_VERSION_DEFAULT="0.33.0" +DH_VERSION_DEFAULT="0.34.1" ######################################################################################################################## # Version Numbers From 05fcdc52a4c875f47c49d58af7cd229dfdcd65e8 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 13:35:01 -0600 Subject: [PATCH 26/51] Addressing the review. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0e82ef35..7a6eb379 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,7 @@ Once in the IDE, you can run queries, create notebooks, and visualize data. You can also run all of the example code below and the more complex examples in [./examples](./examples). To launch the Deephaven IDE, navigate to [http://localhost:10000/ide/](http://localhost:10000/ide/) in your web browser. +Chrome, Edge, Chrome-based, and Firefox browsers are supported. Safari is not supported. How you authenticate will depend upon how authentication is configured. In the examples here, you will use the password `DeephavenRocks!`. From a0bf0b125397ab57c07a7b4014115da01e4590da Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 14:06:19 -0600 Subject: [PATCH 27/51] Addressing the review. --- README.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7a6eb379..f9a849cc 100644 --- a/README.md +++ b/README.md @@ -137,17 +137,19 @@ See the [Deephaven Documentation](https://deephaven.io/core/docs) for details. # Run deephaven-ib -Follow these steps to run a [Deephaven](https://deephaven.io) plus [Interactive Brokers](https://interactivebrokers.com) system. +Follow these steps to run a [Deephaven](https://deephaven.io) plus [Interactive Brokers](https://interactivebrokers.com) system. + +These instructions produce a virtual environment with [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), [Deephaven](https://deephaven.io), and `ibapi` installed. +For more details on using pip-installed Deephaven, see [Deephaven's Installation Guide for pip](https://deephaven.io/core/docs/tutorials/pip-install/). **Windows users need to run the commands in WSL.** -## Setup +## Setup IB To setup and configure the system: -1) Follow the [Deephaven Quick Start Guide](https://deephaven.io/core/docs/tutorials/quickstart/) to get [Deephaven](https://deephaven.io) running. -2) Follow the [TWS Installation Instructions](https://www.interactivebrokers.com/en/trading/tws.php) to get [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) running. -3) Launch [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). -4) In [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php), click on the gear in the +1) Follow the [TWS Installation Instructions](https://www.interactivebrokers.com/en/trading/tws.php) to get [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) running. +2) Launch [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). +3) In [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php), click on the gear in the upper right corner. ![](https://raw.githubusercontent.com/deephaven-examples/deephaven-ib/main/docs/assets/config-gear.png) In `API->Settings`, make sure: @@ -157,17 +159,18 @@ upper right corner. ![](https://raw.githubusercontent.com/deephaven-examples/de Also, note the "Socket port" value. It is needed when connecting [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). ![](https://raw.githubusercontent.com/deephaven-examples/deephaven-ib/main/docs/assets/config-api.png) -5) [For Paper Trading] Log into the [Interactive Brokers Web Interface](https://interactivebrokers.com/). -6) [For Paper Trading] In the [Interactive Brokers Web Interface](https://interactivebrokers.com/), navigate to `Account->Settings->Paper Trading Account` and make sure that "Share real-time market data subscriptions with paper trading account?" is set to true. -7) Once [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) is launched (see [below](#launch)), accept incoming connections to [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). (May not be required for all sessions.) +4) [For Paper Trading] Log into the [Interactive Brokers Web Interface](https://interactivebrokers.com/). +5) [For Paper Trading] In the [Interactive Brokers Web Interface](https://interactivebrokers.com/), navigate to `Account->Settings->Paper Trading Account` and make sure that "Share real-time market data subscriptions with paper trading account?" is set to true. +6) Once [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) is launched (see [below](#launch)), accept incoming connections to [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). (May not be required for all sessions.) ![](https://raw.githubusercontent.com/deephaven-examples/deephaven-ib/main/docs/assets/allow-connections.png) ## Virtual Environment Interactive Brokers does not make their Python wheels available via PyPI, and the wheels are not redistributable. -As a result, installing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) requires a script to build the wheels locally before installation. +As a result, installing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) requires a Python script to build the wheels locally before installation. To keep your development environment clean, the script creates a virtual environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). +Follow the directions below to build and activate the virtual environment using the [./dhib_env.py](./dhib_env.py) script. ### Build the Virtual Environment From d87b35e4e09e051ca304f03644c383d9f097a25c Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:07:00 -0600 Subject: [PATCH 28/51] Support using existing venv or giving a venv a custom name. --- dhib_env.py | 130 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 34 deletions(-) diff --git a/dhib_env.py b/dhib_env.py index 1c176831..3b13d279 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -200,40 +200,13 @@ def pkg_dependencies(path_or_module: Union[str, Path, ModuleType]) -> Dict[str, class Venv: """A virtual environment.""" - def __init__(self, is_release: bool, python: str, dh_version: str, ib_version: str, dh_ib_version: str, - delete_if_exists: bool): + def __init__(self, path: Path): """Create a virtual environment. Args: - is_release: Whether the virtual environment is for a release. - python: The path to the Python executable to use. - dh_version: The version of Deephaven. - ib_version: The version of ibapi. - dh_ib_version: The version of deephaven-ib. - delete_if_exists: Whether to delete the virtual environment if it already exists. + path: The path to the virtual environment. """ - if is_release: - self.path = Path(f"venv-release-dhib={dh_version}").absolute() - else: - self.path = Path(f"venv-dev-dhib={dh_ib_version}-dh={dh_version}-ib={ib_version}").absolute() - - logging.warning(f"Building new virtual environment: {self.path}") - - if delete_if_exists and self.path.exists(): - logging.warning(f"Deleting existing virtual environment: {self.path}") - shutil.rmtree(self.path) - - if self.path.exists(): - logging.error(f"Virtual environment already exists. Please remove it before running this script. venv={self.path}") - raise FileExistsError( - f"Virtual environment already exists. Please remove it before running this script. venv={self.path}") - - logging.warning(f"Creating virtual environment: {self.path}") - shell_exec(f"{python} -m venv {self.path}") - - logging.warning(f"Updating virtual environment: {self.path}") - shell_exec(f"{self.python} -m pip install --upgrade pip") - shell_exec(f"{self.python} -m pip install --upgrade build") + self.path = path @property def python(self) -> str: @@ -257,6 +230,59 @@ def pip_install(self, package: Union[str, Path], version: str = "") -> None: shell_exec(cmd) +def new_venv(path: Path, python: str, delete_if_exists: bool) -> Venv: + """Create a new virtual environment. + + Args: + path: The path to the virtual environment. + python: The path to the Python executable to use. + delete_if_exists: Whether to delete the virtual environment if it already exists. + + Returns: + The new virtual environment. + """ + + logging.warning(f"Building new virtual environment: {path}") + + if delete_if_exists and path.exists(): + logging.warning(f"Deleting existing virtual environment: {path}") + shutil.rmtree(path) + + if path.exists(): + logging.error( + f"Virtual environment already exists. Please remove it before running this script. venv={path}") + raise FileExistsError( + f"Virtual environment already exists. Please remove it before running this script. venv={path}") + + logging.warning(f"Creating virtual environment: {path}") + shell_exec(f"{python} -m venv {path}") + + v = Venv(path) + + logging.warning(f"Updating virtual environment: {path}") + shell_exec(f"{v.python} -m pip install --upgrade pip") + shell_exec(f"{v.python} -m pip install --upgrade build") + + return v + + +def venv_path(is_release: bool, dh_version: str, dh_ib_version: str) -> Path: + """Get the standard path to a new virtual environment. + + Args: + is_release: Whether the virtual environment is for a release. + dh_version: The version of Deephaven. + dh_ib_version: The version of deephaven-ib. + + Returns: + The path to the new virtual environment. + """ + if is_release: + return Path(f"venv-release-dhib={dh_version}").absolute() + else: + return Path(f"venv-dev-dhib={dh_ib_version}-dh={dh_version}").absolute() + + ######################################################################################################################## # IB Wheel ######################################################################################################################## @@ -370,9 +396,21 @@ def cli(): @click.option('--dh_version_exact', default=None, help='The exact version of Deephaven.') @click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') @click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--path_venv', default=None, help='The path to the virtual environment.') +@click.option('--create_venv', default=True, help='Whether to create the virtual environment if it does not already exist.') @click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') @click.option('--install_dhib', default=True, help='Whether to install deephaven-ib. If set to false, the resulting venv can be used to develop deephaven-ib in PyCharm or other development environments.') -def dev(python: str, dh_version: str, dh_version_exact: str, ib_version: str, dh_ib_version: Optional[str], delete_venv: bool, install_dhib: bool): +def dev( + python: str, + dh_version: str, + dh_version_exact: str, + ib_version: str, + dh_ib_version: Optional[str], + path_venv: Optional[str], + create_venv: bool, + delete_venv: bool, + install_dhib: bool +): """Create a development environment.""" logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, dh_version_exact={dh_version_exact}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}") @@ -394,7 +432,15 @@ def dev(python: str, dh_version: str, dh_version_exact: str, ib_version: str, dh version_assert_format(ib_version) version_assert_format(dh_ib_version) - v = Venv(False, python, dh_version, ib_version, dh_ib_version, delete_venv) + if path_venv: + v_path = Path(path_venv).absolute() + else: + v_path = venv_path(False, dh_version, dh_ib_version) + + if create_venv: + v = new_venv(v_path, python, delete_venv) + else: + v = Venv(v_path) ib_wheel = IbWheel(ib_version) ib_wheel.build(v) @@ -419,8 +465,16 @@ def dev(python: str, dh_version: str, dh_version_exact: str, ib_version: str, dh @click.command() @click.option('--python', default="python3", help='The path to the Python executable to use.') @click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--path_venv', default=None, help='The path to the virtual environment.') +@click.option('--create_venv', default=True, help='Whether to create the virtual environment if it does not already exist.') @click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') -def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): +def release( + python: str, + dh_ib_version: Optional[str], + path_venv: Optional[str], + create_venv: bool, + delete_venv: bool +): """Create a release environment.""" logging.warning(f"Creating release environment: python={python} dh_ib_version={dh_ib_version}") @@ -438,7 +492,15 @@ def release(python: str, dh_ib_version: Optional[str], delete_venv: bool): else: dh_ib_version_pip = "" - v = Venv(True, python, dh_version, ib_version, dh_ib_version, delete_venv) + if path_venv: + v_path = Path(path_venv).absolute() + else: + v_path = venv_path(True, dh_version, dh_ib_version) + + if create_venv: + v = new_venv(v_path, python, delete_venv) + else: + v = Venv(v_path) ib_wheel = IbWheel(ib_version) ib_wheel.build(v) From 8e600957dc6c51e035d1b589e95237c5a7791bcf Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:30:12 -0600 Subject: [PATCH 29/51] Support using system python instead of just venvs. --- README.md | 3 ++ dhib_env.py | 129 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 82 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index f9a849cc..9a31d940 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,9 @@ As a result, installing [deephaven-ib](https://github.com/deephaven-examples/dee To keep your development environment clean, the script creates a virtual environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). Follow the directions below to build and activate the virtual environment using the [./dhib_env.py](./dhib_env.py) script. +If you prefer to install directly into your system Python without a virtual environment, +you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). + ### Build the Virtual Environment 1) Install Java 17 and set the appropriate `JAVA_HOME` environment variable. diff --git a/dhib_env.py b/dhib_env.py index 3b13d279..a9e64830 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -197,21 +197,21 @@ def pkg_dependencies(path_or_module: Union[str, Path, ModuleType]) -> Dict[str, ######################################################################################################################## -class Venv: - """A virtual environment.""" +class Pyenv: + """A python environment.""" - def __init__(self, path: Path): - """Create a virtual environment. + def __init__(self, python: str): + """Create a python environment. Args: - path: The path to the virtual environment. + python: The path to the Python executable. """ - self.path = path + self._python = python @property def python(self) -> str: """The path to the Python executable in the virtual environment.""" - return os.path.join(self.path, "bin", "python") + return self._python def pip_install(self, package: Union[str, Path], version: str = "") -> None: """Install a package into the virtual environment. @@ -221,7 +221,7 @@ def pip_install(self, package: Union[str, Path], version: str = "") -> None: version: The version constraint of the package to install. If None, the latest version will be installed. For example, provide "==1.2.3" to install version 1.2.3. """ - logging.warning(f"Installing package in venv: {package}, version: {version}, venv: {self.path}") + logging.warning(f"Installing package in environment: {package}, version: {version}, python: {self.python}") if isinstance(package, Path): package = package.absolute() @@ -230,6 +230,19 @@ def pip_install(self, package: Union[str, Path], version: str = "") -> None: shell_exec(cmd) +class Venv(Pyenv): + """A Python virtual environment.""" + + def __init__(self, path: Path): + """Create a virtual environment. + + Args: + path: The path to the virtual environment. + """ + super().__init__(os.path.join(path, "bin", "python")) + self.path = path + + def new_venv(path: Path, python: str, delete_if_exists: bool) -> Venv: """Create a new virtual environment. @@ -296,7 +309,7 @@ def __init__(self, version: str): """ self.version = version_tuple(version) - def build(self, v: Venv) -> None: + def build(self, pyenv: Pyenv) -> None: """Build the IB wheel. Interactive Brokers does not make their Python wheels available via PyPI, @@ -304,7 +317,7 @@ def build(self, v: Venv) -> None: As a result, we need to build the IB wheel locally. Args: - v: The virtual environment to build the wheel in. + pyenv: The python environment to build the wheel in. """ logging.warning(f"Building IB wheel: {self.version}") @@ -322,18 +335,18 @@ def build(self, v: Venv) -> None: shell_exec("cd build/ib && unzip api.zip") logging.warning(f"Building IB Python API") - shell_exec(f"cd build/ib/IBJts/source/pythonclient && {v.python} -m build --wheel") + shell_exec(f"cd build/ib/IBJts/source/pythonclient && {pyenv.python} -m build --wheel") shell_exec("cp build/ib/IBJts/source/pythonclient/dist/* dist/ib/") - def install(self, v: Venv) -> None: + def install(self, pyenv: Pyenv) -> None: """Install the IB wheel into a virtual environment. Args: - v: The virtual environment to install the wheel into. + pyenv: The python environment to install the wheel into. """ - logging.warning(f"Installing IB wheel in venv: {self.version} {v.path}") + logging.warning(f"Installing IB wheel in python environment: {self.version} python: {pyenv.python}") ver_narrow = version_str(self.version, False) - v.pip_install(Path(f"dist/ib/ibapi-{ver_narrow}-py3-none-any.whl").absolute()) + pyenv.pip_install(Path(f"dist/ib/ibapi-{ver_narrow}-py3-none-any.whl").absolute()) ######################################################################################################################## @@ -353,30 +366,34 @@ def __init__(self, version: str, dh_version: str, ib_version: str): self.dh_version = dh_version self.ib_version = ib_version - def build(self, v: Venv) -> None: + def build(self, pyenv: Pyenv) -> None: """Build the deephaven-ib wheel.""" logging.warning(f"Building deephaven-ib: {self.version}") - shell_exec(f"DH_IB_VERSION={self.version} DH_VERSION={self.dh_version} IB_VERSION={self.ib_version} {v.python} -m build --wheel") + shell_exec(f"DH_IB_VERSION={self.version} DH_VERSION={self.dh_version} IB_VERSION={self.ib_version} {pyenv.python} -m build --wheel") - def install(self, v: Venv) -> None: + def install(self, pyenv: Pyenv) -> None: """Install the deephaven-ib wheel into a virtual environment.""" - logging.warning(f"Installing deephaven-ib in venv: {self.version} {v.path}") - v.pip_install(Path(f"dist/deephaven_ib-{self.version}-py3-none-any.whl").absolute()) + logging.warning(f"Installing deephaven-ib in python environment: {self.version} python: {pyenv.python}") + pyenv.pip_install(Path(f"dist/deephaven_ib-{self.version}-py3-none-any.whl").absolute()) ######################################################################################################################## # Messages ######################################################################################################################## -def success(v: Venv) -> None: +def success(pyenv: Pyenv) -> None: """Print a success message. Args: - v: The virtual environment. + pyenv: The python environment. """ - logging.warning(f"Success! Virtual environment created: {v.path}") - logging.warning(f"Activate the virtual environment with: source {v.path}/bin/activate") - logging.warning(f"Deactivate the virtual environment with: deactivate") + logging.warning("Deephaven-ib environment created successfully.") + logging.warning(f"Python environment: {pyenv.python}") + + if isinstance(pyenv, Venv): + logging.warning(f"Success! Virtual environment created: {pyenv.path}") + logging.warning(f"Activate the virtual environment with: source {pyenv.path}/bin/activate") + logging.warning(f"Deactivate the virtual environment with: deactivate") ######################################################################################################################## @@ -396,6 +413,7 @@ def cli(): @click.option('--dh_version_exact', default=None, help='The exact version of Deephaven.') @click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') @click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--use_venv', default=True, help='Whether to use a python virtual environment or system python.') @click.option('--path_venv', default=None, help='The path to the virtual environment.') @click.option('--create_venv', default=True, help='Whether to create the virtual environment if it does not already exist.') @click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') @@ -406,6 +424,7 @@ def dev( dh_version_exact: str, ib_version: str, dh_ib_version: Optional[str], + use_venv: bool, path_venv: Optional[str], create_venv: bool, delete_venv: bool, @@ -432,45 +451,51 @@ def dev( version_assert_format(ib_version) version_assert_format(dh_ib_version) - if path_venv: - v_path = Path(path_venv).absolute() - else: - v_path = venv_path(False, dh_version, dh_ib_version) + if use_venv: + if path_venv: + v_path = Path(path_venv).absolute() + else: + v_path = venv_path(False, dh_version, dh_ib_version) - if create_venv: - v = new_venv(v_path, python, delete_venv) + if create_venv: + pyenv = new_venv(v_path, python, delete_venv) + else: + pyenv = Venv(v_path) else: - v = Venv(v_path) + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) ib_wheel = IbWheel(ib_version) - ib_wheel.build(v) - ib_wheel.install(v) + ib_wheel.build(pyenv) + ib_wheel.install(pyenv) - v.pip_install("deephaven-server", dh_version_pip) + pyenv.pip_install("deephaven-server", dh_version_pip) if install_dhib: if use_dev: logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version) - dh_ib_wheel.build(v) - dh_ib_wheel.install(v) + dh_ib_wheel.build(pyenv) + dh_ib_wheel.install(pyenv) else: logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") logging.warning(f"*** INSTALLED deephaven-ib MAY BE INCONSISTENT WITH INSTALLED DEPENDENCIES ***") - v.pip_install("deephaven-ib", f"=={dh_ib_version}") + pyenv.pip_install("deephaven-ib", f"=={dh_ib_version}") - success(v) + success(pyenv) @click.command() @click.option('--python', default="python3", help='The path to the Python executable to use.') @click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--use_venv', default=True, help='Whether to use a python virtual environment or system python.') @click.option('--path_venv', default=None, help='The path to the virtual environment.') @click.option('--create_venv', default=True, help='Whether to create the virtual environment if it does not already exist.') @click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') def release( python: str, dh_ib_version: Optional[str], + use_venv: bool, path_venv: Optional[str], create_venv: bool, delete_venv: bool @@ -492,23 +517,27 @@ def release( else: dh_ib_version_pip = "" - if path_venv: - v_path = Path(path_venv).absolute() - else: - v_path = venv_path(True, dh_version, dh_ib_version) + if use_venv: + if path_venv: + v_path = Path(path_venv).absolute() + else: + v_path = venv_path(True, dh_version, dh_ib_version) - if create_venv: - v = new_venv(v_path, python, delete_venv) + if create_venv: + pyenv = new_venv(v_path, python, delete_venv) + else: + pyenv = Venv(v_path) else: - v = Venv(v_path) + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) ib_wheel = IbWheel(ib_version) - ib_wheel.build(v) - ib_wheel.install(v) + ib_wheel.build(pyenv) + ib_wheel.install(pyenv) logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") - v.pip_install("deephaven-ib", dh_ib_version_pip) - success(v) + pyenv.pip_install("deephaven-ib", dh_ib_version_pip) + success(pyenv) cli.add_command(dev) From 0cf841459dbdbfb3b6961e81e0f4e4ad137641d6 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:41:14 -0600 Subject: [PATCH 30/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 7 ++++--- dhib_env.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 99ce8218..18328bc5 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -24,15 +24,16 @@ jobs: uses: actions/checkout@v1 - name: Build run: | - cd ibwhl - docker-compose -f "docker-compose.yml" up --build + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements_dhib_env.txt + python3 dhib_env ibwhl --ib-version ${{ env.IB_VERSION }} find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 with: name: ib-wheels path: | - ibwhl/dist/* + dist/ib/* build-whl: diff --git a/dhib_env.py b/dhib_env.py index a9e64830..3b3740e7 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -406,6 +406,29 @@ def cli(): """A script to build Deephaven-IB virtual environments.""" pass +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') +def ibwheel( + python: str, + ib_version: str, +): + """Create a development environment.""" + logging.warning(f"Creating an ib wheel: python={python}, ib_version={ib_version}") + + version_assert_format(ib_version) + + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) + + ib_wheel = IbWheel(ib_version) + ib_wheel.build(pyenv) + + logging.warning(f"IB wheel created successfully.") + logging.warning(f"IB wheel path: {Path(f'dist/ib/ibapi-{ib_version}-py3-none-any.whl').absolute()}") + + success(pyenv) + @click.command() @click.option('--python', default="python3", help='The path to the Python executable to use.') @@ -540,6 +563,7 @@ def release( success(pyenv) +cli.add_command(ibwheel) cli.add_command(dev) cli.add_command(release) From bc677599c37ed7c59c9dbc278beb1ebec6770e92 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:42:07 -0600 Subject: [PATCH 31/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 18328bc5..14d9b3e7 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -26,7 +26,7 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install -r requirements_dhib_env.txt - python3 dhib_env ibwhl --ib-version ${{ env.IB_VERSION }} + python3 dhib_env.py ibwhl --ib-version ${{ env.IB_VERSION }} find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 From 6dade6e8aa1ef7614f871931ab715ab7de11f13e Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:45:00 -0600 Subject: [PATCH 32/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 14d9b3e7..4b129798 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -24,9 +24,9 @@ jobs: uses: actions/checkout@v1 - name: Build run: | - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements_dhib_env.txt - python3 dhib_env.py ibwhl --ib-version ${{ env.IB_VERSION }} + python3.10 -m pip install --upgrade pip + python3.10 -m pip install -r requirements_dhib_env.txt + python3.10 dhib_env.py ibwhl --python python3.10 --ib-version ${{ env.IB_VERSION }} find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 From 0719fabd846075589b7db7f16b7689d8abc06564 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:46:49 -0600 Subject: [PATCH 33/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 4b129798..c03d1e03 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -24,9 +24,10 @@ jobs: uses: actions/checkout@v1 - name: Build run: | - python3.10 -m pip install --upgrade pip - python3.10 -m pip install -r requirements_dhib_env.txt - python3.10 dhib_env.py ibwhl --python python3.10 --ib-version ${{ env.IB_VERSION }} + python3 --version + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements_dhib_env.txt + python3 dhib_env.py ibwhl --ib-version ${{ env.IB_VERSION }} find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 From d031a832a48aad4531c58d9603528315f97672d2 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:49:28 -0600 Subject: [PATCH 34/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index c03d1e03..95d067b0 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -18,7 +18,7 @@ env: jobs: build-ib-whl: name: Build IB WHL - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v1 @@ -39,7 +39,7 @@ jobs: build-whl: name: Build WHL - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@master - name: Setup Python @@ -81,7 +81,7 @@ jobs: publish-whl: name: Publish WHL - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: [build-whl] if: ${{ github.event_name == 'release' && github.event.action == 'released' }} steps: @@ -100,7 +100,7 @@ jobs: build-sphinx: name: Build Sphinx - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: [build-ib-whl, build-whl] steps: - uses: actions/checkout@v1 @@ -145,7 +145,7 @@ jobs: publish-sphinx: name: Publish Sphinx - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: [build-sphinx] if: ${{ github.event_name == 'release' && github.event.action == 'released' }} steps: @@ -171,7 +171,7 @@ jobs: docker-pip: name: Build and Publish Docker (pip-installed Deephaven) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: [build-ib-whl, build-whl] permissions: contents: read @@ -229,7 +229,7 @@ jobs: docker-dhserver: name: Build and Publish Docker (Deephaven server image) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: [build-ib-whl, build-whl] permissions: contents: read From d95aac9a88611e3e4c3e27611efac36f7e7e17c0 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:55:13 -0600 Subject: [PATCH 35/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 95d067b0..5973a762 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -18,7 +18,7 @@ env: jobs: build-ib-whl: name: Build IB WHL - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v1 @@ -39,7 +39,7 @@ jobs: build-whl: name: Build WHL - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@master - name: Setup Python @@ -81,7 +81,7 @@ jobs: publish-whl: name: Publish WHL - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [build-whl] if: ${{ github.event_name == 'release' && github.event.action == 'released' }} steps: @@ -100,7 +100,7 @@ jobs: build-sphinx: name: Build Sphinx - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [build-ib-whl, build-whl] steps: - uses: actions/checkout@v1 @@ -145,7 +145,7 @@ jobs: publish-sphinx: name: Publish Sphinx - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [build-sphinx] if: ${{ github.event_name == 'release' && github.event.action == 'released' }} steps: @@ -171,7 +171,7 @@ jobs: docker-pip: name: Build and Publish Docker (pip-installed Deephaven) - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [build-ib-whl, build-whl] permissions: contents: read @@ -229,7 +229,7 @@ jobs: docker-dhserver: name: Build and Publish Docker (Deephaven server image) - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [build-ib-whl, build-whl] permissions: contents: read From d0df8e6f9b2b4918caf3888a2eeac8625e1a797e Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:57:05 -0600 Subject: [PATCH 36/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 2 +- dhib_env.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 5973a762..ff478b7e 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -27,7 +27,7 @@ jobs: python3 --version python3 -m pip install --upgrade pip python3 -m pip install -r requirements_dhib_env.txt - python3 dhib_env.py ibwhl --ib-version ${{ env.IB_VERSION }} + python3 dhib_env.py ibwheel --ib-version ${{ env.IB_VERSION }} find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 diff --git a/dhib_env.py b/dhib_env.py index 3b3740e7..e3103e14 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -406,6 +406,7 @@ def cli(): """A script to build Deephaven-IB virtual environments.""" pass + @click.command() @click.option('--python', default="python3", help='The path to the Python executable to use.') @click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') From a559857abc63c71fc3851f19bb5f8f98c988c29c Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 15:59:02 -0600 Subject: [PATCH 37/51] Fix actions to build IB wheels --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index ff478b7e..977f19a5 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -27,7 +27,7 @@ jobs: python3 --version python3 -m pip install --upgrade pip python3 -m pip install -r requirements_dhib_env.txt - python3 dhib_env.py ibwheel --ib-version ${{ env.IB_VERSION }} + python3 dhib_env.py ibwheel --ib_version ${{ env.IB_VERSION }} find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 From 37e43a0be6efbff9bb3660f7ca5f3b351382d727 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 16:00:35 -0600 Subject: [PATCH 38/51] Fix actions to build IB wheels --- requirements_dhib_env.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_dhib_env.txt b/requirements_dhib_env.txt index 35cc9f4b..66662e15 100644 --- a/requirements_dhib_env.txt +++ b/requirements_dhib_env.txt @@ -1,3 +1,4 @@ pkginfo click requests +build \ No newline at end of file From 853ae758d28045d11eb2948773e234cdf20d05b3 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Tue, 7 May 2024 16:11:38 -0600 Subject: [PATCH 39/51] Fix venv-less install. --- dhib_env.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dhib_env.py b/dhib_env.py index e3103e14..4c8d2959 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -417,6 +417,8 @@ def ibwheel( """Create a development environment.""" logging.warning(f"Creating an ib wheel: python={python}, ib_version={ib_version}") + python = Path(python).absolute() if python.startswith("./") else python + version_assert_format(ib_version) logging.warning(f"Using system python: {python}") @@ -457,6 +459,8 @@ def dev( """Create a development environment.""" logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, dh_version_exact={dh_version_exact}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}") + python = Path(python).absolute() if python.startswith("./") else python + if dh_version_exact: if dh_version != DH_VERSION_DEFAULT: raise ValueError(f"Cannot specify both dh_version={dh_version} and dh_version_exact={dh_version_exact}") @@ -527,6 +531,8 @@ def release( """Create a release environment.""" logging.warning(f"Creating release environment: python={python} dh_ib_version={dh_ib_version}") + python = Path(python).absolute() if python.startswith("./") else python + wheel = download_wheel(python, "deephaven_ib", dh_ib_version) deps = pkg_dependencies(wheel) ib_version = deps["ibapi"].replace("==", "") From ca1568aa1810bc230f861f5ececf0e8abc4bb002 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 09:16:00 -0600 Subject: [PATCH 40/51] Improved readme. --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a31d940..863fcb12 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,11 @@ you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). ``` 4) Build a [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) virtual environment: + [./dhib_env.py](./dhib_env.py) is a Python script that builds a Python environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). + The script installs `deephaven-ib`, `ibapi`, and `deephaven` into the environment. + * By default, a virtual environment is created, but this can be disabled with the `--use_venv false` option. + * An existing virtual environment can be used with the `--create_venv false` option. + First, install the dependencies needed to run the script: ```bash python3 -m pip install -r requirements_dhib_env.txt @@ -198,17 +203,20 @@ you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). python3 ./dhib_env.py --help ``` - To install the latest production version from PyPi: + To install the latest production release version of [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) from PyPi: + (This will install the `ibapi` and `deephaven` versions specified in the [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) release.) ```bash python3 ./dhib_env.py release ``` To install the latest development version from source: + (This will install the default `ibapi` and `deephaven` versions.) ```bash python3 ./dhib_env.py dev ``` To create a venv for developing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) in PyCharm: (This will not install deephaven-ib.) + (This will install the default `ibapi` and `deephaven` versions.) ```bash python3 ./dhib_env.py dev --install_dhib false ``` From 0b613d761f54484ec97f86f31c6c4709de8bb84f Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 09:19:55 -0600 Subject: [PATCH 41/51] Improved readme. --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 863fcb12..da1c715f 100644 --- a/README.md +++ b/README.md @@ -168,13 +168,17 @@ upper right corner. ![](https://raw.githubusercontent.com/deephaven-examples/de Interactive Brokers does not make their Python wheels available via PyPI, and the wheels are not redistributable. As a result, installing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) requires a Python script to build the wheels locally before installation. +The script installs `deephaven-ib`, `ibapi`, and `deephaven` into the environment. To keep your development environment clean, the script creates a virtual environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). Follow the directions below to build and activate the virtual environment using the [./dhib_env.py](./dhib_env.py) script. +An existing virtual environment can be used with the `--create_venv false` and `--path_venv ` options. + If you prefer to install directly into your system Python without a virtual environment, you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). + ### Build the Virtual Environment 1) Install Java 17 and set the appropriate `JAVA_HOME` environment variable. @@ -188,11 +192,6 @@ you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). ``` 4) Build a [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) virtual environment: - [./dhib_env.py](./dhib_env.py) is a Python script that builds a Python environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). - The script installs `deephaven-ib`, `ibapi`, and `deephaven` into the environment. - * By default, a virtual environment is created, but this can be disabled with the `--use_venv false` option. - * An existing virtual environment can be used with the `--create_venv false` option. - First, install the dependencies needed to run the script: ```bash python3 -m pip install -r requirements_dhib_env.txt From f57ed16043810651698193d8e5e8717af92ffbe3 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 09:23:39 -0600 Subject: [PATCH 42/51] Improved readme. --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index da1c715f..365b4137 100644 --- a/README.md +++ b/README.md @@ -202,20 +202,17 @@ you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). python3 ./dhib_env.py --help ``` - To install the latest production release version of [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) from PyPi: - (This will install the `ibapi` and `deephaven` versions specified in the [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) release.) + To install the latest production release version of [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) from PyPi plus the release-specified `ibapi` and `deephaven` versions: ```bash python3 ./dhib_env.py release ``` - To install the latest development version from source: - (This will install the default `ibapi` and `deephaven` versions.) + To install the latest development version from source plus the default `ibapi` and `deephaven` versions: ```bash python3 ./dhib_env.py dev ``` - To create a venv for developing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) in PyCharm: (This will not install deephaven-ib.) - (This will install the default `ibapi` and `deephaven` versions.) + To create a venv for developing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) in PyCharm: (This will not install `deephaven-ib`, but it will install the default `ibapi` and `deephaven` versions.) ```bash python3 ./dhib_env.py dev --install_dhib false ``` From 59a70e32c42b4bebac092c014fc9752b84c5bdf4 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 09:24:58 -0600 Subject: [PATCH 43/51] Improved readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 365b4137..bc229337 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). python3 ./dhib_env.py dev --install_dhib false ``` -5) In the logs, take note of where the virtual environment was created. It will be in a directory like `./venv-`. +5) In the logs, take note of where the virtual environment is located. It will be in a directory like `./venv-`. ### Activate the Virtual Environment From 53923a980327856c1216606b7bf567a8b6a15889 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 10:03:34 -0600 Subject: [PATCH 44/51] Build both the ib wheel and dhib wheel --- .github/workflows/build-and-publish.yml | 2 +- dhib_env.py | 59 +++++++++++++++++++++---- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 977f19a5..eaa3a4f5 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -27,7 +27,7 @@ jobs: python3 --version python3 -m pip install --upgrade pip python3 -m pip install -r requirements_dhib_env.txt - python3 dhib_env.py ibwheel --ib_version ${{ env.IB_VERSION }} + python3 dhib_env.py ib-wheel --ib_version ${{ env.IB_VERSION }} find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 diff --git a/dhib_env.py b/dhib_env.py index 4c8d2959..c3bbc42f 100755 --- a/dhib_env.py +++ b/dhib_env.py @@ -338,6 +338,11 @@ def build(self, pyenv: Pyenv) -> None: shell_exec(f"cd build/ib/IBJts/source/pythonclient && {pyenv.python} -m build --wheel") shell_exec("cp build/ib/IBJts/source/pythonclient/dist/* dist/ib/") + @property + def path(self) -> Path: + """The path to the IB wheel.""" + return Path(f"dist/ib/ibapi-{version_str(self.version, False)}-py3-none-any.whl").absolute() + def install(self, pyenv: Pyenv) -> None: """Install the IB wheel into a virtual environment. @@ -346,7 +351,7 @@ def install(self, pyenv: Pyenv) -> None: """ logging.warning(f"Installing IB wheel in python environment: {self.version} python: {pyenv.python}") ver_narrow = version_str(self.version, False) - pyenv.pip_install(Path(f"dist/ib/ibapi-{ver_narrow}-py3-none-any.whl").absolute()) + pyenv.pip_install(self.path) ######################################################################################################################## @@ -371,10 +376,15 @@ def build(self, pyenv: Pyenv) -> None: logging.warning(f"Building deephaven-ib: {self.version}") shell_exec(f"DH_IB_VERSION={self.version} DH_VERSION={self.dh_version} IB_VERSION={self.ib_version} {pyenv.python} -m build --wheel") + @property + def path(self) -> Path: + """The path to the deephaven-ib wheel.""" + return Path(f"dist/deephaven_ib-{self.version}-py3-none-any.whl").absolute() + def install(self, pyenv: Pyenv) -> None: """Install the deephaven-ib wheel into a virtual environment.""" logging.warning(f"Installing deephaven-ib in python environment: {self.version} python: {pyenv.python}") - pyenv.pip_install(Path(f"dist/deephaven_ib-{self.version}-py3-none-any.whl").absolute()) + pyenv.pip_install(self.path) ######################################################################################################################## @@ -410,17 +420,16 @@ def cli(): @click.command() @click.option('--python', default="python3", help='The path to the Python executable to use.') @click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') -def ibwheel( +def ib_wheel( python: str, ib_version: str, ): - """Create a development environment.""" + """Create an ibapi wheel.""" logging.warning(f"Creating an ib wheel: python={python}, ib_version={ib_version}") - python = Path(python).absolute() if python.startswith("./") else python - version_assert_format(ib_version) + python = Path(python).absolute() if python.startswith("./") else python logging.warning(f"Using system python: {python}") pyenv = Pyenv(python) @@ -428,9 +437,40 @@ def ibwheel( ib_wheel.build(pyenv) logging.warning(f"IB wheel created successfully.") - logging.warning(f"IB wheel path: {Path(f'dist/ib/ibapi-{ib_version}-py3-none-any.whl').absolute()}") + logging.warning(f"IB wheel path: {ib_wheel.path}") - success(pyenv) + +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--dh_version', default=DH_VERSION_DEFAULT, help='The version of Deephaven.') +@click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') +@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +def dhib_wheel( + python: str, + dh_version: str, + ib_version: str, + dh_ib_version: Optional[str], +): + """Create a deephaven-ib wheel.""" + logging.warning(f"Creating a deephaven-ib wheel: python={python}, ib_version={ib_version} dh_version={dh_version}, dh_ib_version={dh_ib_version}") + + if dh_ib_version is None: + dh_ib_version = "0.0.0.dev0" + + version_assert_format(ib_version) + version_assert_format(dh_version) + version_assert_format(dh_ib_version) + + python = Path(python).absolute() if python.startswith("./") else python + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) + + logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") + dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version) + dh_ib_wheel.build(pyenv) + + logging.warning(f"Deephaven-ib wheel created successfully.") + logging.warning(f"Deephaven-ib wheel path: {dh_ib_wheel.path}") @click.command() @@ -570,7 +610,8 @@ def release( success(pyenv) -cli.add_command(ibwheel) +cli.add_command(ib_wheel) +cli.add_command(dhib_wheel) cli.add_command(dev) cli.add_command(release) From c2748968859a884310a316264796958d2c8cd591 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 10:29:37 -0600 Subject: [PATCH 45/51] Updated the readme to use the `deephaven` command --- README.md | 56 +++++++++++++++++++++++++++++++-------- examples/run_deephaven.py | 16 ----------- 2 files changed, 45 insertions(+), 27 deletions(-) delete mode 100644 examples/run_deephaven.py diff --git a/README.md b/README.md index bc229337..734501f4 100644 --- a/README.md +++ b/README.md @@ -242,19 +242,48 @@ Once the virtual environment is deactivated, `python` and `pip` will use the sys # Use deephaven-ib -## Start Python +To use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), you need to start a [Deephaven](https://deephaven.io) server and connect to +[IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). +You can optionally use the Deephaven IDE to visualize data and run queries. + +## Start Deephaven + +First, start a [Deephaven](https://deephaven.io) server. This server will be used to process data and run queries. + +The documentation and examples here illustrate using Deephaven's [Pre-Shared Key (PSK) authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/) +with the password `DeephavenRocks!`. Other types of Deephaven authentication can also work. +See the [Deephaven Documentation](https://deephaven.io/core/docs/) for details. + + +### Option 1: Use the `deephaven` command + +The easiest way to start a deephaven server is using `deephaven` on the command line. +The `deephaven` command was added to the virtual environment when it was created. +It is available in [Deephaven](https://deephaven.io) versions `>= 0.34.0`. + +This command will start a deephaven server with 4GB of memory and the password `DeephavenRocks!`. +It will also automatically open the Deephaven IDE in a web browser. -To start Python with the virtual environment, run: ```bash source ./venv-/bin/activate -python +deephaven server --jvm-args "-Xmx4g -Dauthentication.psk=DeephavenRocks! -Dstorage.path=~/.deephaven" ``` -## Start Deephaven -To start Deephaven, run the following python script. It can be found at [./examples/run_deephaven.py](./examples/run_deephaven.py). -This command will start Deephaven with 4GB of memory and the password `DeephavenRocks!`. +### Option 2: Use a Python script +An alternative way to launch a deephaven server is to use a Python script. This works with all versions of +[Deephaven](https://deephaven.io) and can be used to populate the server with queries. +See [Deephaven's Installation Guide for pip](https://deephaven.io/core/docs/tutorials/pip-install/) for more details on +running [Deephaven](https://deephaven.io) this way. + +To start Python with the virtual environment, run: +```bash +source ./venv-/bin/activate +python +``` + +Once Python is running, you can start a deephaven server with the following script: ```python import os from time import sleep @@ -263,19 +292,24 @@ from deephaven_server import Server _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) _server.start() +# You can insert queries here + +# Keep the server running while True: sleep(1) ``` - > :warning: These deephaven server commands **must** be run before importing `deephaven` or `deephaven_ib`. -The documentation and examples here illustrate using Deephaven's [Pre-Shared Key (PSK) authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/) -with the password `DeephavenRocks!`. Other types of Deephaven authentication can also work. -See the [Deephaven Documentation](https://deephaven.io/core/docs/) for details. +At the indicated place in the script, you can put queries that you want to run when the server starts. +This could be code to conenct to [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php), request data, analyze data, visualize data, or trade. +See the examples below for more details. + ## Launch the Deephaven IDE -Once the Deephaven server is started, you can launch the Deephaven IDE. +Once the Deephaven server is started, you can launch the Deephaven IDE. +If you used the `deephaven` command to start the server, the Deephaven IDE will automatically open in your web browser. + The Deephaven IDE is a web-based interface for working with Deephaven. Once in the IDE, you can run queries, create notebooks, and visualize data. You can also run all of the example code below and the more complex examples in [./examples](./examples). diff --git a/examples/run_deephaven.py b/examples/run_deephaven.py deleted file mode 100644 index 3aac6d7d..00000000 --- a/examples/run_deephaven.py +++ /dev/null @@ -1,16 +0,0 @@ - -# This script runs the Deephaven server with the specified port and JVM arguments. -# The server will run until the script is interrupted. -# -# To connect to the Deephaven IDE, navigate to https://localhost:10000 -# The login password is: DeephavenRocks! - -import os -from time import sleep -from deephaven_server import Server - -_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) -_server.start() - -while True: - sleep(1) From f73871df17b1faa6ea02e7f182d714421e498f8f Mon Sep 17 00:00:00 2001 From: Chip Kent <5250374+chipkent@users.noreply.github.com> Date: Wed, 8 May 2024 12:15:53 -0600 Subject: [PATCH 46/51] Update README.md Co-authored-by: Alex Peters <80283343+alexpeters1208@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 734501f4..f6a3a21f 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). python3 ./dhib_env.py release ``` - To install the latest development version from source plus the default `ibapi` and `deephaven` versions: + To install the latest development version of [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) from source plus the default `ibapi` and `deephaven` versions: ```bash python3 ./dhib_env.py dev ``` From f9b1c25093ace3e4d3702ef653e63d304a3cefca Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 12:16:38 -0600 Subject: [PATCH 47/51] Trying to fix sphinx build --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index eaa3a4f5..0eb585a2 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -100,7 +100,7 @@ jobs: build-sphinx: name: Build Sphinx - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 needs: [build-ib-whl, build-whl] steps: - uses: actions/checkout@v1 From 3f8070fc94223ec383918d4b620c8292ad050e46 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 12:18:44 -0600 Subject: [PATCH 48/51] Trying to fix sphinx build --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 0eb585a2..417e6858 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -100,7 +100,7 @@ jobs: build-sphinx: name: Build Sphinx - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: [build-ib-whl, build-whl] steps: - uses: actions/checkout@v1 From 8cd105594bde0f3992b83681fb2b3dc97c6f3306 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 12:23:59 -0600 Subject: [PATCH 49/51] Trying to fix sphinx build --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 417e6858..eaa3a4f5 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -100,7 +100,7 @@ jobs: build-sphinx: name: Build Sphinx - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: [build-ib-whl, build-whl] steps: - uses: actions/checkout@v1 From 7e634eadfd023e1a05e65782d8ddf71d2e32a3de Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 12:26:52 -0600 Subject: [PATCH 50/51] Trying to fix sphinx build --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index eaa3a4f5..38f2c300 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -109,7 +109,7 @@ jobs: sudo apt update sudo apt install -y openjdk-17-jdk - name: Pip installs - run: pip3 install --upgrade sphinx==4.2.0 sphinx-autodoc-typehints furo==2021.10.9 + run: pip3 install --upgrade sphinx~=5.0.0 sphinx-autodoc-typehints furo==2021.10.9 - name: Download IB wheels uses: actions/download-artifact@v3 with: From 2e3b2f4cfa1be252db1c053ac208b4b91c695b94 Mon Sep 17 00:00:00 2001 From: Chip Kent Date: Wed, 8 May 2024 12:34:06 -0600 Subject: [PATCH 51/51] Trying to fix sphinx build --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 38f2c300..acc2f196 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -109,7 +109,7 @@ jobs: sudo apt update sudo apt install -y openjdk-17-jdk - name: Pip installs - run: pip3 install --upgrade sphinx~=5.0.0 sphinx-autodoc-typehints furo==2021.10.9 + run: pip3 install --upgrade sphinx~=7.3.0 sphinx-autodoc-typehints furo==2024.5.6 - name: Download IB wheels uses: actions/download-artifact@v3 with: