diff --git a/.github/workflows/Dockerfile.server b/.github/workflows/Dockerfile.server index 24ed1d85..9f3d288f 100644 --- a/.github/workflows/Dockerfile.server +++ b/.github/workflows/Dockerfile.server @@ -1,6 +1,6 @@ ARG DH_VERSION -FROM ghcr.io/deephaven/server:${DH_VERSION} +FROM ghcr.io/deephaven/server-ui:${DH_VERSION} RUN apt update && \ apt install -y python3-pip python3-venv curl zip && \ diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index af5b3345..6c0387a8 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -12,41 +12,34 @@ on: - released env: - IB_VERSION: 10.19.01 - DH_VERSION: 0.33.3 + IB_VERSION: 10.19.04 + DH_VERSION: 0.34.1 jobs: build-ib-whl: name: Build IB WHL - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel build twine + - name: Checkout + uses: actions/checkout@v1 - name: Build run: | - IB_VERSION_DOWNLOAD=$(echo ${IB_VERSION} | sed 's/[.]//') - echo "Downloading IB API version ${IB_VERSION_DOWNLOAD}" - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION_DOWNLOAD}.zip" - unzip api.zip - cd ./IBJts/source/pythonclient - python -m build + python3 --version + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements_dhib_env.txt + python3 dhib_env.py ib-wheel --ib_version ${{ env.IB_VERSION }} + find . -name \*.whl - name: Archive build artifacts uses: actions/upload-artifact@v2 with: name: ib-wheels path: | - ./IBJts/source/pythonclient/dist/* + dist/ib/* build-whl: name: Build WHL - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@master - name: Setup Python @@ -65,13 +58,14 @@ jobs: then if [ "$GITHUB_REF" = "refs/heads/main" ] then - echo "::set-output name=version::0.0.0.rc" + echo "::set-output name=version::0.0.0-rc.0" else PR_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }') - echo "::set-output name=version::0.0.0.dev${PR_NUMBER}" + echo "::set-output name=version::0.0.0-dev.${PR_NUMBER}" fi else - echo "::set-output name=version::${{ github.event.release.tag_name }}" + SEMVER="${TAG_NAME:1}" + echo "::set-output name=version::${SEMVER}" fi - name: Build env: @@ -88,7 +82,7 @@ jobs: publish-whl: name: Publish WHL - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [build-whl] if: ${{ github.event_name == 'release' && github.event.action == 'released' }} steps: @@ -107,7 +101,7 @@ jobs: build-sphinx: name: Build Sphinx - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [build-ib-whl, build-whl] steps: - uses: actions/checkout@v1 @@ -116,7 +110,7 @@ jobs: sudo apt update sudo apt install -y openjdk-17-jdk - name: Pip installs - run: pip3 install --upgrade sphinx==4.2.0 sphinx-autodoc-typehints furo==2021.10.9 + run: pip3 install --upgrade sphinx~=7.3.0 sphinx-autodoc-typehints furo==2024.5.6 - name: Download IB wheels uses: actions/download-artifact@v3 with: @@ -152,7 +146,7 @@ jobs: publish-sphinx: name: Publish Sphinx - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [build-sphinx] if: ${{ github.event_name == 'release' && github.event.action == 'released' }} steps: @@ -178,7 +172,7 @@ jobs: docker-pip: name: Build and Publish Docker (pip-installed Deephaven) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [build-ib-whl, build-whl] permissions: contents: read @@ -236,7 +230,7 @@ jobs: docker-dhserver: name: Build and Publish Docker (Deephaven server image) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: [build-ib-whl, build-whl] permissions: contents: read diff --git a/.github/workflows/conventional-pr-check.yml b/.github/workflows/conventional-pr-check.yml new file mode 100644 index 00000000..f43cedb0 --- /dev/null +++ b/.github/workflows/conventional-pr-check.yml @@ -0,0 +1,40 @@ +name: 'Conventional PR' + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + pr-check: + runs-on: ubuntu-22.04 + steps: + - uses: amannn/action-semantic-pull-request@v5 + id: lint_pr_title + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: marocchino/sticky-pull-request-comment@v2 + # When the previous steps fails, the workflow would stop. By adding this + # condition you can continue the execution with the populated error message. + if: always() && (steps.lint_pr_title.outputs.error_message != null) + with: + header: pr-title-lint-error + message: | + Pull Request titles must follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/). + Breaking changes that are expected to impact users must include "BREAKING CHANGE: " at the end of the PR description. + + Details: + + ``` + ${{ steps.lint_pr_title.outputs.error_message }} + ``` + + # Delete a previous comment when the issue has been resolved + - if: ${{ steps.lint_pr_title.outputs.error_message == null }} + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: pr-title-lint-error + delete: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 73b04775..28216ca7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .idea venv +venv-* +build dist src/deephaven_ib.egg-info docker/data diff --git a/README.md b/README.md index a23c4022..ce67db2b 100644 --- a/README.md +++ b/README.md @@ -135,22 +135,22 @@ You may want to combine data from other sources with your IB data. [Deephaven]( * [Kafka](https://deephaven.io/core/docs/how-to-guides/kafka-topics/). See the [Deephaven Documentation](https://deephaven.io/core/docs) for details. -Files placed in the `./docker/data/` directory are visible in the Docker container at `/data/`. -See [Access your file system with Docker data volumes](https://deephaven.io/core/docs/conceptual/docker-data-volumes/) for details. - # Run deephaven-ib -Follow these steps to run a [Deephaven](https://deephaven.io) plus [Interactive Brokers](https://interactivebrokers.com) system. +Follow these steps to run a [Deephaven](https://deephaven.io) plus [Interactive Brokers](https://interactivebrokers.com) system. + +These instructions produce a virtual environment with [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), [Deephaven](https://deephaven.io), and `ibapi` installed. +For more details on using pip-installed Deephaven, see [Deephaven's Installation Guide for pip](https://deephaven.io/core/docs/tutorials/pip-install/). -**Windows users need to run the commands in WSL.** +| :exclamation: Windows users _must_ run these commands in WSL. | +|------------------------------------------------------------------| -## Setup +## Setup IB To setup and configure the system: -1) Follow the [Deephaven Quick Start Guide](https://deephaven.io/core/docs/tutorials/quickstart/) to get [Deephaven](https://deephaven.io) running. -2) Follow the [TWS Installation Instructions](https://www.interactivebrokers.com/en/trading/tws.php) to get [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) running. -3) Launch [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). -4) In [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php), click on the gear in the +1) Follow the [TWS Installation Instructions](https://www.interactivebrokers.com/en/trading/tws.php) to get [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) running. +2) Launch [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). +3) In [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php), click on the gear in the upper right corner. ![](https://raw.githubusercontent.com/deephaven-examples/deephaven-ib/main/docs/assets/config-gear.png) In `API->Settings`, make sure: @@ -160,155 +160,174 @@ upper right corner. ![](https://raw.githubusercontent.com/deephaven-examples/de Also, note the "Socket port" value. It is needed when connecting [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). ![](https://raw.githubusercontent.com/deephaven-examples/deephaven-ib/main/docs/assets/config-api.png) -5) [For Paper Trading] Log into the [Interactive Brokers Web Interface](https://interactivebrokers.com/). -6) [For Paper Trading] In the [Interactive Brokers Web Interface](https://interactivebrokers.com/), navigate to `Account->Settings->Paper Trading Account` and make sure that "Share real-time market data subscriptions with paper trading account?" is set to true. -7) Once [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) is launched (see [below](#launch)), accept incoming connections to [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). (May not be required for all sessions.) +4) [For Paper Trading] Log into the [Interactive Brokers Web Interface](https://interactivebrokers.com/). +5) [For Paper Trading] In the [Interactive Brokers Web Interface](https://interactivebrokers.com/), navigate to `Account->Settings->Paper Trading Account` and make sure that "Share real-time market data subscriptions with paper trading account?" is set to true. +6) Once [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) is launched (see [below](#launch)), accept incoming connections to [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). (May not be required for all sessions.) ![](https://raw.githubusercontent.com/deephaven-examples/deephaven-ib/main/docs/assets/allow-connections.png) +## Virtual Environment -## Launch +Interactive Brokers does not make their Python wheels available via PyPI, and the wheels are not redistributable. +As a result, installing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) requires a Python script to build the wheels locally before installation. +The script installs `deephaven-ib`, `ibapi`, and `deephaven` into the environment. -There are multiple ways to launch [Deephaven](https://deephaven.io) with [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) -installed. The launch can either happen via a local installation or via Docker images. +To keep your development environment clean, the script creates a virtual environment for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). +Follow the directions below to build and activate the virtual environment using the [./dhib_env.py](./dhib_env.py) script. -For a Docker Compose example, see [docker/release/](docker/release/). +An existing virtual environment can be used with the `--create_venv false` and `--path_venv ` options. +If you prefer to install directly into your system Python without a virtual environment, +you can use the `--use_venv false` option to [./dhib_env.py](./dhib_env.py). -### (Option 1) Launch pip-installed Deephaven with Docker -- interactive -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed in a Docker container. +### Build the Virtual Environment -1) Create a directory for your data and scripts +1) Install Java 17 and set the appropriate `JAVA_HOME` environment variable. +2) Check out [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) ```bash - mkdir data + git clone git@github.com:deephaven-examples/deephaven-ib.git ``` -2) Create a directory for your Deephaven IDE configuration and notebooks - ```bash - mkdir `pwd`/.deephaven - ``` -3) Launch the system: +3) Change to the deephaven-ib directory: ```bash - # Set jvm_args to the desired JVM memory for Deephaven - docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!']); _server.start()" + cd deephaven-ib ``` -4) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and logging in with the password `DeephavenRocks!`. - - -### (Option 2) Launch pip-installed Deephaven with Docker -- run a script +4) Build a [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) virtual environment: -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed in a Docker container. -This is a good option for production scenarios where scripts need to be run and related data needs to be visualized. - -1) Create a directory for your data and scripts + First, install the dependencies needed to run the script: ```bash - mkdir data - # your_script.py must begin with: "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!']); _server.start()" - # Set jvm_args to the desired JVM memory for Deephaven - cp path/to/your_script.py data/your_script.py + python3 -m pip install -r requirements_dhib_env.txt ``` -2) Create a directory for your Deephaven IDE configuration and notebooks - ```bash - mkdir `pwd`/.deephaven - ``` -3) Launch the system: + + To see all options: ```bash - # Set jvm_args to the desired JVM memory for Deephaven - docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 ghcr.io/deephaven-examples/deephaven-ib python3 -i /data/your_script.py + python3 ./dhib_env.py --help ``` -4) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and logging in with the password `DeephavenRocks!`. - -### (Option 3) Launch pip-installed Deephaven with a local installation (No Docker) -- interactive - -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed directly on your local system, without Docker. - -It is possible to use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) without docker, but this is a -new feature and has not been well tested. To do this: -1) Install `ibapi`: + To install the latest production release version of [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) from PyPi plus the release-specified `ibapi` and `deephaven` versions: ```bash - # pip installed version of ibapi is too old. You must download and install a more recent version. - export IB_VERSION=1019.01 - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" - unzip api.zip - cd ./IBJts/source/pythonclient - python3 setup.py install + python3 ./dhib_env.py release ``` -2) Install Java 11 and set the appropriate `JAVA_HOME` environment variable. -3) Install [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib): + + To install the latest development version of [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) from source plus the default `ibapi` and `deephaven` versions: ```bash - pip3 install --upgrade pip setuptools wheel - pip3 install deephaven-ib + python3 ./dhib_env.py dev ``` -4) Launch the system: + + To create a venv for developing [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) in PyCharm: (This will not install `deephaven-ib`, but it will install the default `ibapi` and `deephaven` versions.) ```bash - # Set jvm_args to the desired JVM memory for Deephaven - # Deephaven IDE configuration and notebooks are stored to ~/.deephaven - python3 -i -c "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]); _server.start()" + python3 ./dhib_env.py dev --install_dhib false ``` -5) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and log in with the password `DeephavenRocks!`. -6) Use `host="localhost"` for the hostname in the examples (Windows WSL uses `host="host.docker.internal"`, since WSL is built on Docker.) + +5) In the logs, take note of where the virtual environment is located. It will be in a directory like `./venv-`. + +### Activate the Virtual Environment -### (Option 4) Launch pip-installed Deephaven with a local installation (No Docker) -- run a script +To activate the virtual environment: +```bash +source ./venv-/bin/activate +``` -The pip-installed Deephaven uses a lightweight Deephaven installation that is installed using pip. In this case, -the pip-installed Deephaven system is installed directly on your local system, without Docker. -This is a good option for production scenarios where scripts need to be run and related data needs to be visualized. +Once the virtual environment is activated, `python` and `pip` will use the virtual environment's Python and packages -- +including everything needed to run [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). -It is possible to use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) without docker, but this is a -new feature and has not been well tested. To do this: -1) Install `ibapi`: - ```bash - # pip installed version of ibapi is too old. You must download and install a more recent version. - export IB_VERSION=1016.01 - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" - unzip api.zip - cd ./IBJts/source/pythonclient - python3 setup.py install - ``` -2) Install Java 11 and set the appropriate `JAVA_HOME` environment variable. -3) Install [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib): - ```bash - pip3 install --upgrade pip setuptools wheel - pip3 install deephaven-ib - ``` -4) Launch the system and execute a custom script: - ```bash - # your_script.py must begin with: "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]); _server.start()" - # Deephaven IDE configuration and notebooks are stored to ~/.deephaven - # Set jvm_args to the desired JVM memory for Deephaven - python3 -i your_script.py - ``` -5) Launch the [Deephaven IDE](https://github.com/deephaven/deephaven-core/blob/main/README.md#run-deephaven-ide) by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in a browser and log in with the password `DeephavenRocks!`. -6) Use `host=localhost` for the hostname in the examples +### Deactivate the Virtual Environment + +To deactivate the virtual environment: +```bash +deactivate +``` + +Once the virtual environment is deactivated, `python` and `pip` will use the system's Python and packages. +[deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) will not be available. + + +# Use deephaven-ib + +To use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), you need to start a [Deephaven](https://deephaven.io) server and connect to +[IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). +You can optionally use the Deephaven IDE to visualize data and run queries. + +## Start Deephaven -# Authentication +First, start a [Deephaven](https://deephaven.io) server. This server will be used to process data and run queries. The documentation and examples here illustrate using Deephaven's [Pre-Shared Key (PSK) authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/) with the password `DeephavenRocks!`. Other types of Deephaven authentication can also work. See the [Deephaven Documentation](https://deephaven.io/core/docs/) for details. -Common ways to authenticate are: -* [Anonymous Authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-anon/) -* [Pre-Shared Key (PSK) Authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/) -* [Username / Password Authentication](https://deephaven.io/core/docs/how-to-guides/authentication/auth-uname-pw/) -# Use deephaven-ib +### Option 1: Use the `deephaven` command + +The easiest way to start a deephaven server is using `deephaven` on the command line. +The `deephaven` command was added to the virtual environment when it was created. +It is available in [Deephaven](https://deephaven.io) versions `>= 0.34.0`. + +This command will start a deephaven server with 4GB of memory and the password `DeephavenRocks!`. +It will also automatically open the Deephaven IDE in a web browser. + +```bash +source ./venv-/bin/activate +deephaven server --jvm-args "-Xmx4g -Dauthentication.psk=DeephavenRocks! -Dstorage.path=~/.deephaven" +``` + + +### Option 2: Use a Python script + +An alternative way to launch a deephaven server is to use a Python script. This works with all versions of +[Deephaven](https://deephaven.io) and can be used to populate the server with queries. +See [Deephaven's Installation Guide for pip](https://deephaven.io/core/docs/tutorials/pip-install/) for more details on +running [Deephaven](https://deephaven.io) this way. + +To start Python with the virtual environment, run: +```bash +source ./venv-/bin/activate +python +``` + +Once Python is running, you can start a deephaven server with the following script: +```python +import os +from time import sleep +from deephaven_server import Server + +_server = Server(port=10000, jvm_args=['-Xmx4g','-Dauthentication.psk=DeephavenRocks!','-Dstorage.path=' + os.path.expanduser('~/.deephaven')]) +_server.start() + +# You can insert queries here + +# Keep the server running +while True: + sleep(1) +``` +> :warning: These deephaven server commands **must** be run before importing `deephaven` or `deephaven_ib`. + +At the indicated place in the script, you can put queries that you want to run when the server starts. +This could be code to conenct to [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php), request data, analyze data, visualize data, or trade. +See the examples below for more details. + -To use [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), you will need to open the [Deephaven Web IDE](http://localhost:10000/ide/) -by navigating to [http://localhost:10000/ide/](http://localhost:10000/ide/) in your web browser. How you authenticate -will depend upon how authentication is configured. In the examples here, you will use the password `DeephavenRocks!`. +## Launch the Deephaven IDE + +Once the Deephaven server is started, you can launch the Deephaven IDE. +If you used the `deephaven` command to start the server, the Deephaven IDE will automatically open in your web browser. + +The Deephaven IDE is a web-based interface for working with Deephaven. +Once in the IDE, you can run queries, create notebooks, and visualize data. +You can also run all of the example code below and the more complex examples in [./examples](./examples). + +To launch the Deephaven IDE, navigate to [http://localhost:10000/ide/](http://localhost:10000/ide/) in your web browser. +Chrome, Edge, Chrome-based, and Firefox browsers are supported. Safari is not supported. +How you authenticate will depend upon how authentication is configured. +In the examples here, you will use the password `DeephavenRocks!`. -The following commands can be executed in the console. ## Connect to TWS All [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) sessions need to first create a client for interacting with [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php). -`host` is the computer to connect to. When using [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) inside +`host` is the computer to connect to. When using [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) locally, `host` is usually set to `localhost`. +When using [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) inside of Docker, `host` should be set to `host.docker.internal`. `port` is the network port [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) @@ -332,7 +351,7 @@ For a read-write session that allows trading: ```python import deephaven_ib as dhib -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, read_only=False) +client = dhib.IbSessionTws(host="localhost", port=7497, read_only=False) client.connect() ``` @@ -340,7 +359,7 @@ For a read-only session that does not allow trading: ```python import deephaven_ib as dhib -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, read_only=True) +client = dhib.IbSessionTws(host="localhost", port=7497, read_only=True) client.connect() ``` @@ -348,7 +367,7 @@ For a read-only financial advisor (FA) session that does not allow trading: ```python import deephaven_ib as dhib -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, read_only=True, is_fa=True) +client = dhib.IbSessionTws(host="localhost", port=7497, read_only=True, is_fa=True) client.connect() ``` @@ -679,7 +698,11 @@ If you can not solve your problems through either the `errors` table or through ### `Takes N positional arguments but M were given` -You may encounter an error that looks like: `Takes N positional arguments but M were given`. If you see a problem like this, your `ibapi` version does not match the version needed by [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). The [`ibapi` version in PyPI](https://pypi.org/project/ibapi/) is ancient and appears to have been abandoned by [Interactive Brokers](https://www.interactivebrokers.com/). Currently [Interactive Brokers](https://www.interactivebrokers.com/) is delivering `ibapi` either via the [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) download or via git. +You may encounter an error that looks like: `Takes N positional arguments but M were given`. +If you see a problem like this, your `ibapi` version does not match the version needed by [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib). +The [`ibapi` version in PyPI](https://pypi.org/project/ibapi/) is ancient and appears to have been abandoned by [Interactive Brokers](https://www.interactivebrokers.com/). +Currently [Interactive Brokers](https://www.interactivebrokers.com/) is delivering `ibapi` either via the [IB Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) +download or via git. To check your `ibapi` version: ```python @@ -687,17 +710,8 @@ import ibapi print(ibapi.__version__) ``` -If your version is `9.x`, it is the old version from [PyPI](https://pypi.org/project/ibapi/). To install the required `ibapi` version: -```bash -# pip installed version of ibapi is too old. You must download and install a more recent version. -export IB_VERSION=1019.01 -curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION}.zip" -unzip api.zip -cd ./IBJts/source/pythonclient -python3 setup.py install -``` - -Note that the `ibapi` API is very unstable. You likely need the exact version mentioned here for [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib) to function. If you want a better installation experience and more flexability with the `ibapi` version, reach out to [Interactive Brokers](https://www.interactivebrokers.com/) and let them know that you want them to start publishing the latest `ibapi` versions to [PyPI](https://pypi.org). +The `ibapi` API is very unstable. If your version does not exactly match the version needed by [deephaven-ib](https://github.com/deephaven-examples/deephaven-ib), +you will need to install the correct version. Regenerate your virtual environment as described above. # Examples diff --git a/dhib_env.py b/dhib_env.py new file mode 100755 index 00000000..c3bbc42f --- /dev/null +++ b/dhib_env.py @@ -0,0 +1,619 @@ +#!/usr/bin/env python3 + +""" A script to build a virtual environment for Deephaven-IB development or release.""" + +import atexit +import logging +import os +import re +import shutil +from pathlib import Path +from types import ModuleType +from typing import Optional, Dict, Union +import click +import pkginfo +import requests + +IB_VERSION_DEFAULT="10.19.04" +DH_VERSION_DEFAULT="0.34.1" + +######################################################################################################################## +# Version Numbers +######################################################################################################################## + + +def version_tuple(version: str) -> tuple[int, ...]: + """Convert a version string to a tuple of integers. + + Args: + version: The version string to convert. + + Returns: + A tuple of integers representing the version. + """ + return tuple(map(int, (version.split(".")))) + + +def version_str(version: tuple[int, ...], wide: bool) -> str: + """Convert a version tuple to a string. + + Args: + version: The version tuple to convert. + wide: Whether to use a wide format that includes leading zeros. + + Returns: + A string representing the version. + """ + if wide: + return ".".join(f"{x:02d}" for x in version) + else: + return ".".join(map(str, version)) + + +def version_assert_format(version: str) -> None: + """Assert that a version string is formatted correctly. + + Args: + version: The version string to check. + + Raises: + ValueError: If the version string is not formatted correctly. + """ + if not version: + raise ValueError("Version string is empty.") + + # check if the version string is in semver format + pattern1 = re.compile(r"^([0-9]\d*)\.([0-9]\d*)\.([0-9]\d*)$") + pattern2 = re.compile(r"^([0-9]\d*)\.([0-9]\d*)\.([0-9]\d*)\.dev([0-9]\d*)$") + is_semver = bool(pattern1.match(version)) or bool(pattern2.match(version)) + + if not is_semver: + raise ValueError(f"Version string is not in semver format: {version}") + + +######################################################################################################################## +# Shell +######################################################################################################################## + + +def shell_exec(cmd: str) -> None: + """Execute a shell command. + + Args: + cmd: The command to execute. + """ + logging.warning(f"Executing shell command: {cmd}") + e = os.system(cmd) + + if e != 0: + raise Exception(f"Error executing shell command: {cmd}") + + +######################################################################################################################## +# URL +######################################################################################################################## + + +def url_download(url: str, path: Union[str, Path]) -> None: + """Download a file from a URL. + + Args: + url: The URL to download from. + path: The path to save the downloaded file to. + """ + logging.warning(f"Downloading file: {url}, path: {path}") + response = requests.get(url) + response.raise_for_status() + + with open(path, "wb") as f: + f.write(response.content) + + +######################################################################################################################## +# Package Query Functions +######################################################################################################################## + + +def delete_file_on_exit(file_path: Union[str, Path]) -> None: + """Register a file to be deleted on program exit.""" + + def delete_file(): + if os.path.exists(file_path): + os.remove(file_path) + logging.debug(f"{file_path} has been deleted.") + + atexit.register(delete_file) + + +def download_wheel(python: str, package: str, version: Optional[str], delete_on_exit: bool = True) -> Path: + """Download a wheel file for a package with a specific version. + + Args: + python: The path to the Python executable to use. + package: The name of the package to download. + version: The version of the package to download. If None, the latest version will be downloaded. + delete_on_exit: Whether to delete the wheel file on program exit. + + Returns: + The path of the downloaded wheel file. + + Raises: + subprocess.CalledProcessError: If the download process fails. + """ + logging.warning(f"Downloading wheel for package: {package}, version: {version}, delete_on_exit: {delete_on_exit}") + + if not version: + logging.warning(f"Determining latest version of package: {package}") + response = requests.get(f"https://pypi.org/pypi/{package}/json") + response.raise_for_status() + version = response.json()["info"]["version"] + + ver = f"=={version}" if version else "" + shell_exec(f"{python} -m pip download {package}{ver} --no-deps") + p = Path(f"{package}-{version}-py3-none-any.whl").absolute() + + if delete_on_exit: + delete_file_on_exit(str(p)) + + return p + + +def pkg_dependencies(path_or_module: Union[str, Path, ModuleType]) -> Dict[str, Optional[str]]: + """Get the dependencies of a package. + + Args: + path_or_module: The path to the package or the module object. + + Returns: + A dictionary containing the dependencies of the package and their version specifications. + """ + + if isinstance(path_or_module, Path): + path_or_module = str(path_or_module) + + meta = pkginfo.get_metadata(path_or_module) + + if not meta: + raise ValueError(f"Package could not be found: {path_or_module}") + + rst = {} + + for req in meta.requires_dist: + s = req.split(" ") + name = s[0] + + if len(s) > 1: + version = s[1].strip("()") + else: + version = None + + rst[name] = version + + return rst + + +######################################################################################################################## +# Venv +######################################################################################################################## + + +class Pyenv: + """A python environment.""" + + def __init__(self, python: str): + """Create a python environment. + + Args: + python: The path to the Python executable. + """ + self._python = python + + @property + def python(self) -> str: + """The path to the Python executable in the virtual environment.""" + return self._python + + def pip_install(self, package: Union[str, Path], version: str = "") -> None: + """Install a package into the virtual environment. + + Args: + package: The name of the package to install. + version: The version constraint of the package to install. If None, the latest version will be installed. + For example, provide "==1.2.3" to install version 1.2.3. + """ + logging.warning(f"Installing package in environment: {package}, version: {version}, python: {self.python}") + + if isinstance(package, Path): + package = package.absolute() + + cmd = f"""{self.python} -m pip install {package}{version}""" + shell_exec(cmd) + + +class Venv(Pyenv): + """A Python virtual environment.""" + + def __init__(self, path: Path): + """Create a virtual environment. + + Args: + path: The path to the virtual environment. + """ + super().__init__(os.path.join(path, "bin", "python")) + self.path = path + + +def new_venv(path: Path, python: str, delete_if_exists: bool) -> Venv: + """Create a new virtual environment. + + Args: + path: The path to the virtual environment. + python: The path to the Python executable to use. + delete_if_exists: Whether to delete the virtual environment if it already exists. + + Returns: + The new virtual environment. + """ + + logging.warning(f"Building new virtual environment: {path}") + + if delete_if_exists and path.exists(): + logging.warning(f"Deleting existing virtual environment: {path}") + shutil.rmtree(path) + + if path.exists(): + logging.error( + f"Virtual environment already exists. Please remove it before running this script. venv={path}") + raise FileExistsError( + f"Virtual environment already exists. Please remove it before running this script. venv={path}") + + logging.warning(f"Creating virtual environment: {path}") + shell_exec(f"{python} -m venv {path}") + + v = Venv(path) + + logging.warning(f"Updating virtual environment: {path}") + shell_exec(f"{v.python} -m pip install --upgrade pip") + shell_exec(f"{v.python} -m pip install --upgrade build") + + return v + + +def venv_path(is_release: bool, dh_version: str, dh_ib_version: str) -> Path: + """Get the standard path to a new virtual environment. + + Args: + is_release: Whether the virtual environment is for a release. + dh_version: The version of Deephaven. + dh_ib_version: The version of deephaven-ib. + + Returns: + The path to the new virtual environment. + """ + if is_release: + return Path(f"venv-release-dhib={dh_version}").absolute() + else: + return Path(f"venv-dev-dhib={dh_ib_version}-dh={dh_version}").absolute() + + +######################################################################################################################## +# IB Wheel +######################################################################################################################## + +class IbWheel: + def __init__(self, version: str): + """Create an IB wheel. + + Args: + version: The version of the IB wheel. + """ + self.version = version_tuple(version) + + def build(self, pyenv: Pyenv) -> None: + """Build the IB wheel. + + Interactive Brokers does not make their Python wheels available via PyPI, + and the wheels are not redistributable. + As a result, we need to build the IB wheel locally. + + Args: + pyenv: The python environment to build the wheel in. + """ + logging.warning(f"Building IB wheel: {self.version}") + + shutil.rmtree("build/ib", ignore_errors=True) + shutil.rmtree("dist/ib", ignore_errors=True) + + os.makedirs("build/ib", exist_ok=True) + os.makedirs("dist/ib", exist_ok=True) + + logging.warning(f"Downloading IB API version {self.version}") + ver_ib = f"{self.version[0]:02d}{self.version[1]:02d}.{self.version[2]:02d}" + url_download(f"https://interactivebrokers.github.io/downloads/twsapi_macunix.{ver_ib}.zip", "build/ib/api.zip") + + logging.warning(f"Unzipping IB API") + shell_exec("cd build/ib && unzip api.zip") + + logging.warning(f"Building IB Python API") + shell_exec(f"cd build/ib/IBJts/source/pythonclient && {pyenv.python} -m build --wheel") + shell_exec("cp build/ib/IBJts/source/pythonclient/dist/* dist/ib/") + + @property + def path(self) -> Path: + """The path to the IB wheel.""" + return Path(f"dist/ib/ibapi-{version_str(self.version, False)}-py3-none-any.whl").absolute() + + def install(self, pyenv: Pyenv) -> None: + """Install the IB wheel into a virtual environment. + + Args: + pyenv: The python environment to install the wheel into. + """ + logging.warning(f"Installing IB wheel in python environment: {self.version} python: {pyenv.python}") + ver_narrow = version_str(self.version, False) + pyenv.pip_install(self.path) + + +######################################################################################################################## +# deephaven-ib +######################################################################################################################## + +class DhIbWheel: + def __init__(self, version: str, dh_version: str, ib_version: str): + """Create a deephaven-ib wheel. + + Args: + version: The version of the deephaven-ib wheel. + dh_version: The version of Deephaven. + ib_version: The version of ibapi. + """ + self.version = version + self.dh_version = dh_version + self.ib_version = ib_version + + def build(self, pyenv: Pyenv) -> None: + """Build the deephaven-ib wheel.""" + logging.warning(f"Building deephaven-ib: {self.version}") + shell_exec(f"DH_IB_VERSION={self.version} DH_VERSION={self.dh_version} IB_VERSION={self.ib_version} {pyenv.python} -m build --wheel") + + @property + def path(self) -> Path: + """The path to the deephaven-ib wheel.""" + return Path(f"dist/deephaven_ib-{self.version}-py3-none-any.whl").absolute() + + def install(self, pyenv: Pyenv) -> None: + """Install the deephaven-ib wheel into a virtual environment.""" + logging.warning(f"Installing deephaven-ib in python environment: {self.version} python: {pyenv.python}") + pyenv.pip_install(self.path) + + +######################################################################################################################## +# Messages +######################################################################################################################## + +def success(pyenv: Pyenv) -> None: + """Print a success message. + + Args: + pyenv: The python environment. + """ + logging.warning("Deephaven-ib environment created successfully.") + logging.warning(f"Python environment: {pyenv.python}") + + if isinstance(pyenv, Venv): + logging.warning(f"Success! Virtual environment created: {pyenv.path}") + logging.warning(f"Activate the virtual environment with: source {pyenv.path}/bin/activate") + logging.warning(f"Deactivate the virtual environment with: deactivate") + + +######################################################################################################################## +# Click CLI +######################################################################################################################## + + +@click.group() +def cli(): + """A script to build Deephaven-IB virtual environments.""" + pass + + +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') +def ib_wheel( + python: str, + ib_version: str, +): + """Create an ibapi wheel.""" + logging.warning(f"Creating an ib wheel: python={python}, ib_version={ib_version}") + + version_assert_format(ib_version) + + python = Path(python).absolute() if python.startswith("./") else python + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) + + ib_wheel = IbWheel(ib_version) + ib_wheel.build(pyenv) + + logging.warning(f"IB wheel created successfully.") + logging.warning(f"IB wheel path: {ib_wheel.path}") + + +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--dh_version', default=DH_VERSION_DEFAULT, help='The version of Deephaven.') +@click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') +@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +def dhib_wheel( + python: str, + dh_version: str, + ib_version: str, + dh_ib_version: Optional[str], +): + """Create a deephaven-ib wheel.""" + logging.warning(f"Creating a deephaven-ib wheel: python={python}, ib_version={ib_version} dh_version={dh_version}, dh_ib_version={dh_ib_version}") + + if dh_ib_version is None: + dh_ib_version = "0.0.0.dev0" + + version_assert_format(ib_version) + version_assert_format(dh_version) + version_assert_format(dh_ib_version) + + python = Path(python).absolute() if python.startswith("./") else python + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) + + logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") + dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version) + dh_ib_wheel.build(pyenv) + + logging.warning(f"Deephaven-ib wheel created successfully.") + logging.warning(f"Deephaven-ib wheel path: {dh_ib_wheel.path}") + + +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--dh_version', default=DH_VERSION_DEFAULT, help='The version of Deephaven.') +@click.option('--dh_version_exact', default=None, help='The exact version of Deephaven.') +@click.option('--ib_version', default=IB_VERSION_DEFAULT, help='The version of ibapi.') +@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--use_venv', default=True, help='Whether to use a python virtual environment or system python.') +@click.option('--path_venv', default=None, help='The path to the virtual environment.') +@click.option('--create_venv', default=True, help='Whether to create the virtual environment if it does not already exist.') +@click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') +@click.option('--install_dhib', default=True, help='Whether to install deephaven-ib. If set to false, the resulting venv can be used to develop deephaven-ib in PyCharm or other development environments.') +def dev( + python: str, + dh_version: str, + dh_version_exact: str, + ib_version: str, + dh_ib_version: Optional[str], + use_venv: bool, + path_venv: Optional[str], + create_venv: bool, + delete_venv: bool, + install_dhib: bool +): + """Create a development environment.""" + logging.warning(f"Creating development environment: python={python} dh_version={dh_version}, dh_version_exact={dh_version_exact}, ib_version={ib_version}, dh_ib_version={dh_ib_version}, delete_vm_if_exists={delete_venv}") + + python = Path(python).absolute() if python.startswith("./") else python + + if dh_version_exact: + if dh_version != DH_VERSION_DEFAULT: + raise ValueError(f"Cannot specify both dh_version={dh_version} and dh_version_exact={dh_version_exact}") + + dh_version = dh_version_exact + dh_version_pip = f"=={dh_version}" + else: + dh_version_pip = f"~={dh_version}" + + use_dev = dh_ib_version is None + + if dh_ib_version is None: + dh_ib_version = "0.0.0.dev0" + + version_assert_format(dh_version) + version_assert_format(ib_version) + version_assert_format(dh_ib_version) + + if use_venv: + if path_venv: + v_path = Path(path_venv).absolute() + else: + v_path = venv_path(False, dh_version, dh_ib_version) + + if create_venv: + pyenv = new_venv(v_path, python, delete_venv) + else: + pyenv = Venv(v_path) + else: + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) + + ib_wheel = IbWheel(ib_version) + ib_wheel.build(pyenv) + ib_wheel.install(pyenv) + + pyenv.pip_install("deephaven-server", dh_version_pip) + + if install_dhib: + if use_dev: + logging.warning(f"Building deephaven-ib from source: {dh_ib_version}") + dh_ib_wheel = DhIbWheel(dh_ib_version, dh_version, ib_version) + dh_ib_wheel.build(pyenv) + dh_ib_wheel.install(pyenv) + else: + logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") + logging.warning(f"*** INSTALLED deephaven-ib MAY BE INCONSISTENT WITH INSTALLED DEPENDENCIES ***") + pyenv.pip_install("deephaven-ib", f"=={dh_ib_version}") + + success(pyenv) + + +@click.command() +@click.option('--python', default="python3", help='The path to the Python executable to use.') +@click.option('--dh_ib_version', default=None, help='The version of deephaven-ib.') +@click.option('--use_venv', default=True, help='Whether to use a python virtual environment or system python.') +@click.option('--path_venv', default=None, help='The path to the virtual environment.') +@click.option('--create_venv', default=True, help='Whether to create the virtual environment if it does not already exist.') +@click.option('--delete_venv', default=False, help='Whether to delete the virtual environment if it already exists.') +def release( + python: str, + dh_ib_version: Optional[str], + use_venv: bool, + path_venv: Optional[str], + create_venv: bool, + delete_venv: bool +): + """Create a release environment.""" + logging.warning(f"Creating release environment: python={python} dh_ib_version={dh_ib_version}") + + python = Path(python).absolute() if python.startswith("./") else python + + wheel = download_wheel(python, "deephaven_ib", dh_ib_version) + deps = pkg_dependencies(wheel) + ib_version = deps["ibapi"].replace("==", "") + dh_version = deps["deephaven-server"].replace("==", "").replace("~=", "") + + version_assert_format(dh_version) + version_assert_format(ib_version) + + if dh_ib_version: + version_assert_format(dh_ib_version) + dh_ib_version_pip = f"=={dh_ib_version}" + else: + dh_ib_version_pip = "" + + if use_venv: + if path_venv: + v_path = Path(path_venv).absolute() + else: + v_path = venv_path(True, dh_version, dh_ib_version) + + if create_venv: + pyenv = new_venv(v_path, python, delete_venv) + else: + pyenv = Venv(v_path) + else: + logging.warning(f"Using system python: {python}") + pyenv = Pyenv(python) + + ib_wheel = IbWheel(ib_version) + ib_wheel.build(pyenv) + ib_wheel.install(pyenv) + + logging.warning(f"Installing deephaven-ib from PyPI: {dh_ib_version}") + pyenv.pip_install("deephaven-ib", dh_ib_version_pip) + success(pyenv) + + +cli.add_command(ib_wheel) +cli.add_command(dhib_wheel) +cli.add_command(dev) +cli.add_command(release) + +if __name__ == '__main__': + cli() diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile deleted file mode 100644 index 4b0fa0d4..00000000 --- a/docker/dev/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# -# A docker image from the current repository. -# - -FROM ubuntu:22.04 - -ARG DH_IB_VERSION=0.0.0dev -ARG IB_VERSION -ARG DH_VERSION - -# Install requirements - -RUN apt update && \ - apt install -y openjdk-17-jdk && \ - ln -s /usr/lib/jvm/java-17-openjdk-*/ /usr/lib/jvm/java-17-openjdk && \ - apt install --yes git python3-venv python3-pip curl unzip && \ - pip3 install --upgrade pip setuptools wheel build - -ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk - -# Build and install ibapi - -RUN mkdir /build && \ - cd /build && \ - IB_VERSION_DOWNLOAD=$(echo ${IB_VERSION} | sed 's/[.]//') && \ - echo "Downloading IB API version ${IB_VERSION_DOWNLOAD}" && \ - curl -o ./api.zip "https://interactivebrokers.github.io/downloads/twsapi_macunix.${IB_VERSION_DOWNLOAD}.zip" && \ - unzip api.zip && \ - cd ./IBJts/source/pythonclient && \ - python3 setup.py install && \ - cd / && \ - rm -rf /build - -# Build and install deephaven-ib - -COPY ./build /build - -RUN cd /build && \ - python3 -m build && \ - pip3 install dist/*.whl && \ - rm -rf /build - -CMD python3 diff --git a/docker/dev/README.md b/docker/dev/README.md deleted file mode 100644 index 67d6194f..00000000 --- a/docker/dev/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Build Docker Image From Current Deephaven-IB Checkout - -This directory contains the ingredients to build Docker images from the current, checked-out deephaven-ib repository. - -This is useful when doing local development or when official images are not available for your platform. - -In general, you will want to use the officially released images at [https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib](https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib). - -## Build Image - -```bash -./build.sh -``` - -## Run image in interactive mode - -```bash -docker compose up -``` - -or - -```bash -# Set jvm_args to the desired JVM memory for Deephaven -docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 deephaven-examples/deephaven-ib:dev python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()" -``` - diff --git a/docker/dev/build.sh b/docker/dev/build.sh deleted file mode 100755 index a3295363..00000000 --- a/docker/dev/build.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# Build a docker image from the current repository. -# - -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd ${__dir} - -rm -rf build -mkdir build -rsync -av ../.. build --exclude docker -rm -rf build/dist - -if [ -z "$DH_VERSION" ]; then - echo "DH_VERSION must be set" - exit 1 -fi - -IB_VERSION_DEFAULT=10.19.01 - -if [ -z "$IB_VERSION" ]; then - echo "Using default IB_VERSION=${IB_VERSION_DEFAULT}" - IB_VERSION=${IB_VERSION_DEFAULT} -fi - -docker build --build-arg DH_VERSION=${DH_VERSION} --build-arg IB_VERSION=${IB_VERSION} -t deephaven-examples/deephaven-ib:dev -f Dockerfile . - -rm -rf build \ No newline at end of file diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml deleted file mode 100644 index 818ec6a6..00000000 --- a/docker/dev/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.4" - -services: - server: - image: deephaven-examples/deephaven-ib:dev - environment: - # Deephaven PSK password: See https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/#setting-your-own-key - DH_PASSWORD: ${DH_PASSWORD:?DH_PASSWORD not set} - command: python3 -i -c "import os; from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g',f'-Dauthentication.psk={os.getenv(\"DH_PASSWORD\")}']); _server.start()" - stdin_open: true - tty: true - ports: - - "${DEEPHAVEN_PORT:-10000}:10000" - volumes: - - ./data:/data - - ~/.deephaven:/storage - diff --git a/docker/release/README.md b/docker/release/README.md deleted file mode 100644 index b5fe0069..00000000 --- a/docker/release/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Build Docker Image From The Most Recent pip-installable Deephaven-IB Release - -This directory contains the ingredients to launch a prebuilt deephaven-ib image. - -In general, you will want to use the officially released images at [https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib](https://github.com/deephaven-examples/deephaven-ib/pkgs/container/deephaven-ib). - -## Build Image - -```bash -./build.sh -``` - -## Run image in interactive mode - -```bash -DH_IB_VERSION= docker compose up -``` - -```bash -# Set jvm_args to the desired JVM memory for Deephaven -docker run -it -v data:/data -v `pwd`/.deephaven:/storage -p 10000:10000 deephaven-examples/deephaven-ib: python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g']); _server.start()" -``` diff --git a/docker/release/docker-compose.yml b/docker/release/docker-compose.yml deleted file mode 100644 index 8b60e9a7..00000000 --- a/docker/release/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.4" - -services: - server: - image: ghcr.io/deephaven-examples/deephaven-ib:${DH_IB_VERSION:?DH_IB_VERSION not set} - environment: - # Deephaven PSK password: See https://deephaven.io/core/docs/how-to-guides/authentication/auth-psk/#setting-your-own-key - DH_PASSWORD: ${DH_PASSWORD:?DH_PASSWORD not set} - command: python3 -i -c "from deephaven_server import Server; _server = Server(port=10000, jvm_args=['-Xmx4g',f'-Dauthentication.psk={os.getenv(\"DH_PASSWORD\")}']); _server.start()" - stdin_open: true - tty: true - ports: - - "${DEEPHAVEN_PORT:-10000}:10000" - volumes: - - ./data:/data - - ~/.deephaven:/storage - diff --git a/examples/example_all_functionality.py b/examples/example_all_functionality.py index 58a37b79..2ff8783d 100644 --- a/examples/example_all_functionality.py +++ b/examples/example_all_functionality.py @@ -1,3 +1,6 @@ + +# Run this example in the Deephaven IDE Console + from typing import Dict from ibapi.contract import Contract @@ -14,7 +17,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, client_id=0, download_short_rates=True, read_only=False) +client = dhib.IbSessionTws(host="localhost", port=7497, client_id=0, download_short_rates=True, read_only=False) print(f"IsConnected: {client.is_connected()}") client.connect() @@ -129,8 +132,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "20240119" - contract.strike = 138.5 + contract.lastTradeDateOrContractMonth = "20260116" + contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" rst["option_1"] = contract @@ -163,8 +166,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "FOP" contract.exchange = "CME" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "202312" - contract.strike = 4700 + contract.lastTradeDateOrContractMonth = "202409" + contract.strike = 5000 contract.right = "C" contract.multiplier = "50" rst["futureoption_1"] = contract @@ -173,14 +176,14 @@ def get_contracts() -> Dict[str, Contract]: contract = Contract() # enter CUSIP as symbol - contract.symbol = "912828C57" + contract.symbol = "084664BL4" contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" rst["bond_1"] = contract contract = Contract() - contract.conId = 147554578 + contract.conId = 577489715 contract.exchange = "SMART" rst["bond_2"] = contract @@ -316,7 +319,7 @@ def get_contracts() -> Dict[str, Contract]: # enter CUSIP as symbol contract = Contract() -contract.symbol = "IBCID411964960" +contract.conId = 505885457 contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" @@ -456,8 +459,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" -contract.lastTradeDateOrContractMonth = "20240119" -contract.strike = 138.5 +contract.lastTradeDateOrContractMonth = "20260116" +contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" diff --git a/examples/example_beta_calc.py b/examples/example_beta_calc.py index 41534f64..2c14cc7f 100644 --- a/examples/example_beta_calc.py +++ b/examples/example_beta_calc.py @@ -1,3 +1,6 @@ + +# Run this example in the Deephaven IDE Console + ## Set the API port. Default port numbers are: # 7496 - Trader Workstation, real trading # 4001 - IB Gateway, real trading @@ -13,7 +16,7 @@ else: read_only_api = True -client = dhib.IbSessionTws(host="host.docker.internal", port=API_PORT, read_only=read_only_api) +client = dhib.IbSessionTws(host="localhost", port=API_PORT, read_only=read_only_api) client.connect() if client.is_connected(): diff --git a/examples/example_market_data.py b/examples/example_market_data.py index bd45faa2..c5df7499 100644 --- a/examples/example_market_data.py +++ b/examples/example_market_data.py @@ -1,3 +1,6 @@ + +# Run this example in the Deephaven IDE Console + from ibapi.contract import Contract import deephaven_ib as dhib @@ -6,7 +9,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, download_short_rates=False) +client = dhib.IbSessionTws(host="localhost", port=7497, download_short_rates=False) client.connect() # Makes all tables global variables so that they are displayed in the user interface diff --git a/examples/example_market_maker.py b/examples/example_market_maker.py index 5318c897..f4424676 100644 --- a/examples/example_market_maker.py +++ b/examples/example_market_maker.py @@ -1,4 +1,6 @@ +# Run this example in the Deephaven IDE Console + from ibapi.contract import Contract from ibapi.order import Order @@ -19,7 +21,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, client_id=0, download_short_rates=False, read_only=False) +client = dhib.IbSessionTws(host="localhost", port=7497, client_id=0, download_short_rates=False, read_only=False) print(f"IsConnected: {client.is_connected()}") client.connect() diff --git a/examples/example_option_risk.py b/examples/example_option_risk.py index a9513d36..3c4bf0ce 100644 --- a/examples/example_option_risk.py +++ b/examples/example_option_risk.py @@ -1,5 +1,7 @@ # Compute real-time risk scenarios for a portfolio of options +# Run this example in the Deephaven IDE Console + import math from ibapi.contract import Contract from deephaven.constants import NULL_DOUBLE @@ -11,7 +13,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, download_short_rates=False) +client = dhib.IbSessionTws(host="localhost", port=7497, download_short_rates=False) client.connect() print("==============================================================================================================") diff --git a/examples/example_overview_image.py b/examples/example_overview_image.py index 034a2564..c705d3a5 100644 --- a/examples/example_overview_image.py +++ b/examples/example_overview_image.py @@ -1,10 +1,13 @@ + +# Run this example in the Deephaven IDE Console + import deephaven_ib as dhib print("==============================================================================================================") print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497) +client = dhib.IbSessionTws(host="localhost", port=7497) client.connect() from ibapi.contract import Contract diff --git a/examples/example_query_and_plot.py b/examples/example_query_and_plot.py index c374b18f..82f29e7f 100644 --- a/examples/example_query_and_plot.py +++ b/examples/example_query_and_plot.py @@ -1,10 +1,13 @@ + +# Run this example in the Deephaven IDE Console + import deephaven_ib as dhib print("==============================================================================================================") print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497) +client = dhib.IbSessionTws(host="localhost", port=7497) client.connect() from ibapi.contract import Contract diff --git a/examples/example_read_only_functionality.py b/examples/example_read_only_functionality.py index 423c9462..a8ef9144 100644 --- a/examples/example_read_only_functionality.py +++ b/examples/example_read_only_functionality.py @@ -1,3 +1,6 @@ + +# Run this example in the Deephaven IDE Console + from typing import Dict from ibapi.contract import Contract @@ -10,7 +13,7 @@ print("==== ** Accept the connection in TWS **") print("==============================================================================================================") -client = dhib.IbSessionTws(host="host.docker.internal", port=7497, client_id=0, download_short_rates=True, read_only=True) +client = dhib.IbSessionTws(host="localhost", port=7497, client_id=0, download_short_rates=True, read_only=True) print(f"IsConnected: {client.is_connected()}") client.connect() @@ -125,8 +128,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "20240119" - contract.strike = 138.5 + contract.lastTradeDateOrContractMonth = "20260116" + contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" rst["option_1"] = contract @@ -159,8 +162,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "FOP" contract.exchange = "CME" contract.currency = "USD" - contract.lastTradeDateOrContractMonth = "202312" - contract.strike = 4700 + contract.lastTradeDateOrContractMonth = "202409" + contract.strike = 5000 contract.right = "C" contract.multiplier = "50" rst["futureoption_1"] = contract @@ -169,14 +172,14 @@ def get_contracts() -> Dict[str, Contract]: contract = Contract() # enter CUSIP as symbol - contract.symbol = "912828C57" + contract.symbol = "084664BL4" contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" rst["bond_1"] = contract contract = Contract() - contract.conId = 147554578 + contract.conId = 577489715 contract.exchange = "SMART" rst["bond_2"] = contract @@ -312,7 +315,7 @@ def get_contracts() -> Dict[str, Contract]: # enter CUSIP as symbol contract = Contract() -contract.symbol = "IBCID411964960" +contract.conId = 505885457 contract.secType = "BOND" contract.exchange = "SMART" contract.currency = "USD" @@ -452,8 +455,8 @@ def get_contracts() -> Dict[str, Contract]: contract.secType = "OPT" contract.exchange = "BOX" contract.currency = "USD" -contract.lastTradeDateOrContractMonth = "20240119" -contract.strike = 138.5 +contract.lastTradeDateOrContractMonth = "20260116" +contract.strike = 170.0 contract.right = "C" contract.multiplier = "100" diff --git a/requirements_dhib_env.txt b/requirements_dhib_env.txt new file mode 100644 index 00000000..66662e15 --- /dev/null +++ b/requirements_dhib_env.txt @@ -0,0 +1,4 @@ +pkginfo +click +requests +build \ No newline at end of file diff --git a/setup.py b/setup.py index 21f345ea..86776dee 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import os +import re import setuptools @@ -8,17 +9,58 @@ dh_ib_version = os.getenv("DH_IB_VERSION") if not dh_ib_version: - raise Exception("deephaven-ib version must be set via the DH_IB_VERSION environment varialble.") + raise Exception("deephaven-ib version must be set via the DH_IB_VERSION environment variable.") dh_version = os.getenv("DH_VERSION") if not dh_version: - raise Exception("deephaven version must be set via the DH_VERSION environment varialble.") + raise Exception("deephaven version must be set via the DH_VERSION environment variable.") ib_version = os.getenv("IB_VERSION") if not ib_version: - raise Exception("ibapi version must be set via the IB_VERSION environment varialble.") + raise Exception("ibapi version must be set via the IB_VERSION environment variable.") + + +def is_valid_semver(version: str, allow_zero_prefix: bool=False): + """ + Checks if a string is in valid semver format. + + Args: + version: The version string to validate. + allow_zero_prefix: Allow zero prefixes + + Returns: + True if the string is in valid semver format, False otherwise. + """ + if allow_zero_prefix: + pattern = r'^([0-9]\d*)\.([0-9]\d*)\.([0-9]\d*)(-([0-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.([0-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$' + else: + pattern = r'^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$' + + return bool(re.match(pattern, version)) + + +def version_assert_format(version: str, allow_zero_prefix: bool=False) -> None: + """Assert that a version string is formatted correctly. + + Args: + version: The version string to check. + allow_zero_prefix: Allow zero prefixes + + Raises: + ValueError: If the version string is not formatted correctly. + """ + if not version: + raise ValueError("Version string is empty.") + + if not is_valid_semver(version, allow_zero_prefix=allow_zero_prefix): + raise ValueError(f"Version string is not in semver format: {version}") + + +version_assert_format(dh_ib_version) +version_assert_format(dh_version) +version_assert_format(ib_version, allow_zero_prefix=True) setuptools.setup( name="deephaven_ib", @@ -47,9 +89,9 @@ ], package_dir={"": "src"}, packages=setuptools.find_packages(where="src"), - python_requires=">=3.6", + python_requires=">=3.10", install_requires=[ - f"deephaven-server=={dh_version}", + f"deephaven-server~={dh_version}", "pandas", f"ibapi=={ib_version}", "lxml", diff --git a/src/deephaven_ib/__init__.py b/src/deephaven_ib/__init__.py index a26af40b..c0bd83fb 100644 --- a/src/deephaven_ib/__init__.py +++ b/src/deephaven_ib/__init__.py @@ -550,7 +550,7 @@ def annotate_ticks(t): return rst - def deephaven_ib_float_value(s: str) -> Optional[float]: + def deephaven_ib_float_value(s: Optional[str]) -> Optional[float]: if not s: return NULL_DOUBLE @@ -638,7 +638,7 @@ def deephaven_ib_parse_note(note:str, key:str) -> Optional[str]: "orders_status": tables_raw["raw_orders_status"] \ .last_by("PermId") \ .move_columns_up(["ReceiveTime", "PermId", "ClientId", "OrderId", "ParentId"]), - "bars_historical": annotate_ticks(tables_raw["raw_bars_historical"]).last_by(["Request", "Timestamp", "ContractId"]), + "bars_historical": annotate_ticks(tables_raw["raw_bars_historical"]).last_by(["RequestId", "Timestamp", "ContractId"]), "bars_realtime": annotate_ticks(tables_raw["raw_bars_realtime"]), "ticks_efp": annotate_ticks(tables_raw["raw_ticks_efp"]), "ticks_generic": annotate_ticks(tables_raw["raw_ticks_generic"]), @@ -692,8 +692,12 @@ def get_registered_contract(self, contract: Contract) -> RegisteredContract: """ self._assert_connected() - cd = self._client.contract_registry.request_contract_details_blocking(contract) - return RegisteredContract(query_contract=contract, contract_details=cd) + + try: + cd = self._client.contract_registry.request_contract_details_blocking(contract) + return RegisteredContract(query_contract=contract, contract_details=cd) + except Exception as e: + raise Exception(f"Error getting registered contract: contract={contract} {e}") def request_contracts_matching(self, pattern: str) -> Request: """Request contracts matching a pattern. Results are returned in the ``contracts_matching`` table. diff --git a/src/deephaven_ib/_internal/short_rates.py b/src/deephaven_ib/_internal/short_rates.py index ede01ec7..7252fc5b 100644 --- a/src/deephaven_ib/_internal/short_rates.py +++ b/src/deephaven_ib/_internal/short_rates.py @@ -71,7 +71,8 @@ def write(self, line: str) -> None: def load_short_rates() -> Table: """Downloads the short rates from the IB FTP site and returns them as a table.""" - host: str = "ftp3.interactivebrokers.com" + # See: https://www.ibkrguides.com/kb/article-2024.htm + host: str = "ftp2.interactivebrokers.com" user: str = "shortstock" with ftplib.FTP(host=host, user=user) as ftp, IBFtpWriter() as p: