Merge remote-tracking branch 'origin/main' into 116-queue-state-updates

docker/config/deephaven.prop

@@ -4,7 +4,6 @@ deephaven.console.type=python
 
 # Add all plugins that you want installed here
 deephaven.jsPlugins.@deephaven/js-plugin-ui=/opt/deephaven/config/plugins/plugins/ui/src/js
-deephaven.jsPlugins.@deephaven/js-plugin-plotly-express=/opt/deephaven/config/plugins/plugins/plotly-express/src/js
 
 # Anonymous authentication so we don't need to put in a password
 AuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler

plugins/matplotlib/setup.cfg

@@ -3,7 +3,7 @@ name = deephaven-plugin-matplotlib
 description = Deephaven Plugin for matplotlib
 long_description = file: README.md
 long_description_content_type = text/markdown
-version = attr:deephaven.plugin.matplotlib.__version__
+version = 0.4.0.dev0
 url = https://github.com/deephaven/deephaven-plugins
 project_urls =
     Source Code = https://github.com/deephaven/deephaven-plugins

plugins/matplotlib/src/deephaven/plugin/matplotlib/__init__.py

@@ -6,8 +6,6 @@
 from matplotlib.animation import Animation
 import itertools
 
-__version__ = "0.3.0.dev0"
-
 
 def _init_theme():
     # Set the Deephaven style globally.

plugins/matplotlib/src/deephaven/plugin/matplotlib/figure_type.py

@@ -2,8 +2,11 @@
 from weakref import WeakKeyDictionary, WeakSet
 from matplotlib.figure import Figure
 from deephaven.plugin.object_type import Exporter, FetchOnlyObjectType
+from deephaven.execution_context import get_exec_ctx
 from threading import Timer
 from deephaven.liveness_scope import liveness_scope
+from deephaven import input_table, new_table
+from deephaven.column import string_col, int_col
 
 # Name of the matplotlib figure object that was export
 NAME = "matplotlib.figure.Figure"
@@ -49,34 +52,33 @@ def call_it():
 # width: The width of panel displaying the figure
 # height: The height of the panel displaying the figure
 def _make_input_table(figure):
-    from deephaven import new_table
-    from deephaven.column import string_col, int_col
-    import jpy
-
-    input_table = None
+    input_t = None
     revision = 0
 
+    # Need to track the execution context used so we can use it again when updating the revision
+    exec_ctx = get_exec_ctx()
+
     t = new_table(
         [
             string_col("key", ["revision", "width", "height"]),
             int_col("value", [revision, 640, 480]),
         ]
     )
-    input_table = jpy.get_type(
-        "io.deephaven.engine.table.impl.util.KeyedArrayBackedMutableTable"
-    ).make(t.j_table, "key")
+
+    input_t = input_table(init_table=t, key_cols=["key"])
 
     # TODO: Add listener to input table to update figure width/height
 
     @debounce(0.1)
     def update_revision():
-        nonlocal revision
-        revision = revision + 1
-        input_table.getAttribute("InputTable").add(
-            new_table(
-                [string_col("key", ["revision"]), int_col("value", [revision])]
-            ).j_table
-        )
+        with exec_ctx:
+            nonlocal revision
+            revision = revision + 1
+            input_t.add(
+                new_table(
+                    [string_col("key", ["revision"]), int_col("value", [revision])]
+                )
+            )
 
     def handle_figure_update(self, value):
         # Check if we're already drawing this figure, and the stale callback was triggered because of our call to savefig
@@ -86,7 +88,7 @@ def handle_figure_update(self, value):
 
     figure.stale_callback = handle_figure_update
 
-    return input_table
+    return input_t
 
 
 def _get_input_table(figure):
@@ -117,8 +119,8 @@ def is_type(self, object) -> bool:
         return isinstance(object, Figure)
 
     def to_bytes(self, exporter: Exporter, figure: Figure) -> bytes:
-        with liveness_scope() as scope:
-            input_table = _get_input_table(figure)
-            exporter.reference(input_table)
-            scope.preserve(input_table)
+        with liveness_scope() as scope, get_exec_ctx():
+            input_t = _get_input_table(figure)
+            exporter.reference(input_t)
+            scope.preserve(input_t)
         return _export_figure(figure)

plugins/plotly-express/test/deephaven/plot/express/preprocess/test_TimePreprocessor.py

@@ -40,14 +40,15 @@ def test_time_preprocessor(self):
 
         expected_df = pd.DataFrame(
             {
-                "Start": ["2021-07-04 12:00:00"],
-                "End": ["2021-07-04 13:00:00"],
+                "Start": ["2021-07-04 12:00:00+00:00"],
+                "End": ["2021-07-04 13:00:00+00:00"],
                 "Category": ["A"],
                 "x_diff": [3600000.0],
             }
         )
         expected_df["Start"] = pd.to_datetime(expected_df["Start"])
         expected_df["End"] = pd.to_datetime(expected_df["End"])
+        expected_df["Category"] = expected_df["Category"].astype("string")
         expected_df["x_diff"] = expected_df["x_diff"].astype("Float64")
 
         new_df = dhpd.to_pandas(new_table)

plugins/ui/DESIGN.md

@@ -1388,22 +1388,24 @@ ui_table.selection_mode(mode: SelectionMode) -> UITable
 ##### sort
 
 Provide the default sort that will be used by the UI.
+Can use Deephaven [SortDirection](https://deephaven.io/core/pydoc/code/deephaven.html#deephaven.SortDirection) used in 
+a table [sort](https://deephaven.io/core/docs/reference/table-operations/sort/) operation or`"ASC"` or `"DESC"`.
 
 ###### Syntax
 
 ```py
 ui_table.sort(
     order_by: str | Sequence[str],
-    order: SortDirection | Sequence[SortDirection] | None = None
+    order: TableSortDirection | Sequence[TableSortDirection] | None = None
 ) -> UITable
 ```
 
 ###### Parameters
 
-| Parameter   | Type                                               | Description                                                                                                 |
-| ----------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
-| `by`        | `str \| Sequence[str]`                             | The column(s) to sort by. May be a single column name, or a list of column names.                           |
-| `direction` | `SortDirection \| Sequence[SortDirection] \| None` | The sort direction(s) to use. If provided, that must match up with the columns provided. Defaults to "ASC". |
+| Parameter   | Type                                                         | Description                                                                                                 |
+| ----------- |--------------------------------------------------------------| ----------------------------------------------------------------------------------------------------------- |
+| `by`        | `str \| Sequence[str]`                                       | The column(s) to sort by. May be a single column name, or a list of column names.                           |
+| `direction` | `TableSortDirection \| Sequence[TableSortDirection] \| None` | The sort direction(s) to use. If provided, that must match up with the columns provided. Defaults to "ASC". |
 
 #### ui.fragment
 
@@ -1458,6 +1460,8 @@ use_table_listener(
 ##### use_table_data
 
 Capture the data in a table. If the table is still loading, a sentinel value will be returned.
+A transform function can be used to transform the data from a pandas Dataframe to a custom object, but this should
+not be used to perform large filtering operations.
 Data should already be filtered to the desired rows and columns before passing to this hook as it is best to filter before data is retrieved.
 Use functions such as [head](https://deephaven.io/core/docs/reference/table-operations/filter/head/) or [slice](https://deephaven.io/core/docs/reference/table-operations/filter/slice/) to retrieve specific rows and functions such
 as [select or view](https://deephaven.io/core/docs/how-to-guides/use-select-view-update/) to retrieve specific columns.
@@ -1467,16 +1471,20 @@ as [select or view](https://deephaven.io/core/docs/how-to-guides/use-select-view
 ```py
 use_table_data(
     table: Table,
-    sentinel: Sentinel = None
-) -> TableData | Sentinel:
+    sentinel: Sentinel = None,
+    transform: Callable[
+        [pd.DataFrame | Sentinel, bool], TransformedData | Sentinel
+    ] = None,
+) -> TableData | Sentinel | TransformedData:
 ```
 
 ###### Parameters
 
-| Parameter  | Type       | Description                                                                  |
-| ---------- | ---------- | ---------------------------------------------------------------------------- |
-| `table`    | `Table`    | The table to retrieve data from.                                             |
-| `sentinel` | `Sentinel` | A sentinel value to return if the viewport is still loading. Default `None`. |
+| Parameter          | Type                                                                      | Description                                                                                                                                                                                                                             |
+|--------------------|---------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `table`            | `Table`                                                                   | The table to retrieve data from.                                                                                                                                                                                                        |
+| `sentinel`         | `Sentinel`                                                                | A sentinel value to return if the viewport is still loading. Default `None`.                                                                                                                                                            |
+| `transform`        | `Callable[[pd.DataFrame \| Sentinel, bool], TransformedData \| Sentinel]` | A function to transform the data from a pandas Dataframe to a custom object. The function takes a pandas dataframe or `Sentinel` as the first value and as a second value `bool` that is `True` if the the first value is the sentinel. |
 
 ##### use_column_data
 
@@ -1599,8 +1607,9 @@ RowIndex = int | None
 SearchMode = Literal["SHOW", "HIDE", "DEFAULT"]
 SelectionMode = Literal["CELL", "ROW", "COLUMN"]
 Sentinel = Any
-SortDirection = Literal["ASC", "DESC"]
+TableSortDirection = Union[Literal["ASC", "DESC"], SortDirection]
 TableData = dict[ColumnName, ColumnData]
+TransformedData = Any
 
 # Set a filter for a dashboard. Filter will apply to all items with a matching column/type, except for items specified in the `exclude_ids` parameter
 class DashboardFilter(TypedDict):

plugins/ui/examples/README.md

@@ -710,7 +710,7 @@ t = time_table("PT1S").update(formulas=["X=i"]).tail(5)
 monitor = monitor_changed_data(t)
 ```
 
-![Stock Rollup](assets/change_monitor.png)
+![Change Monitor](assets/change_monitor.png)
 
 ## Tabs
 
@@ -747,6 +747,73 @@ def table_tabs(source):
 tt = table_tabs(stocks)
 ```
 
+## Using Table Data Hooks
+
+There are five different hooks that can be used to get data from a table:
+
+1. `use_table_data`: Returns a dictionary of rows and columns from the table.
+2. `use_row_data`: Returns a single row from the table as a dictionary
+3. `use_row_list`: Returns a single row from the table as a list
+4. `use_column_data`: Returns a single column from the table as a list
+5. `use_cell_data`: Returns a single cell from the table
+
+In this example, the hooks are used to display various pieces of information about LIZARD trades.
+
+```python
+import deephaven.ui as ui
+from deephaven.table import Table
+from deephaven import time_table, agg
+import deephaven.plot.express as dx
+
+stocks = dx.data.stocks()
+
+
+@ui.component
+def watch_lizards(source: Table):
+
+    sold_lizards = source.where(["side in `sell`", "sym in `LIZARD`"])
+    exchange_count_table = sold_lizards.view(["exchange"]).count_by(
+        "count", by=["exchange"]
+    )
+    last_sell_table = sold_lizards.tail(1)
+    max_size_and_price_table = sold_lizards.agg_by([agg.max_(cols=["size", "price"])])
+    last_ten_sizes_table = sold_lizards.view("size").tail(10)
+    average_sell_table = (
+        sold_lizards.view(["size", "dollars"])
+        .tail(100)
+        .sum_by()
+        .view("average = dollars/size")
+    )
+
+    exchange_count = ui.use_table_data(exchange_count_table)
+    last_sell = ui.use_row_data(last_sell_table)
+    max_size_and_price = ui.use_row_list(max_size_and_price_table)
+    last_ten_sizes = ui.use_column_data(last_ten_sizes_table)
+    average_sell = ui.use_cell_data(average_sell_table)
+
+    exchange_count_view = ui.view(f"Exchange counts {exchange_count}")
+    last_sell_view = ui.view(f"Last Sold LIZARD: {last_sell}")
+    max_size_and_price_view = ui.view(f"Max size and max price: {max_size_and_price}")
+    last_ten_sizes_view = ui.view(f"Last Ten Sizes: {last_ten_sizes}")
+    average_sell_view = ui.view(f"Average LIZARD price: {average_sell}")
+
+    return ui.flex(
+        exchange_count_view,
+        last_sell_view,
+        max_size_and_price_view,
+        last_ten_sizes_view,
+        average_sell_view,
+        margin=10,
+        gap=10,
+        direction="column",
+    )
+
+
+watch = watch_lizards(stocks)
+```
+
+![Table Hooks](assets/table_hooks.png)
+
 ## Multi-threading
 
 State updates must be called from the render thread. All callbacks are automatically called from the render thread, but sometimes you will need to do some long-running operations asynchronously. You can use the `use_render_queue` hook to run a callback on the render thread. In this example, we create a form that takes a URL as input, and loads the CSV file from another thread before updating the state on the current thread.