Merge branch 'main' into documentation

.dockerignore

@@ -41,4 +41,10 @@ playwright/.cache/
 **/.gitignore
 
 # Tests are copied to the docker container, as it modifies them
-tests/
+tests/
+
+# Playwright
+test-results/
+playwright-report/
+blob-report/
+playwright/.cache/

.github/workflows/e2e.yml

@@ -0,0 +1,43 @@
+name: End-to-end Tests
+
+on:
+  push:
+    branches:
+      - main
+      - 'release/**'
+  pull_request:
+    branches:
+      - main
+      - 'release/**'
+
+jobs:
+  e2e-test:
+    runs-on: ubuntu-22.04
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run tests
+        run: "./tools/run_docker.sh e2e-tests"
+
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 90
+
+      - name: Dump server logs
+        if: failure()
+        run: docker logs deephaven-plugins > /tmp/server-log.txt
+
+      - name: Upload server logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-logs
+          path: /tmp/server-log.txt

.github/workflows/modified-plugin.yml

@@ -11,6 +11,8 @@ on:
       - 'matplotlib-v*'
       - 'json-v*'
       - 'ui-v*'
+      - 'utilities-v*'
+      - 'packaging-v*'
 
 jobs:
   changes:
@@ -27,11 +29,27 @@ jobs:
         id: filter
         with:
           filters: |
-            plotly-express: plugins/plotly-express/**
-            plotly: plugins/plotly/**
-            matplotlib: plugins/matplotlib/**
-            json: plugins/json/**
-            ui: plugins/ui/**
+            plotly-express:
+              - plugins/plotly-express/**
+              - .github/workflows/test-*.yml
+            plotly:
+              - plugins/plotly/**
+              - .github/workflows/test-*.yml
+            matplotlib:
+              - plugins/matplotlib/**
+              - .github/workflows/test-*.yml
+            json:
+              - plugins/json/**
+              - .github/workflows/test-*.yml
+            ui:
+              - plugins/ui/**
+              - .github/workflows/test-*.yml
+            utilities:
+              - plugins/utilities/**
+              - .github/workflows/test-*.yml
+            packaging:
+              - plugins/packaging/**
+              - .github/workflows/test-*.yml
 
   # Test all python packages that have been modified individually
   test-python:

.github/workflows/release-python-package.yml

@@ -28,10 +28,18 @@ jobs:
           node-version: '18.x'
           registry-url: 'https://registry.npmjs.org'
 
+      - name: Check file existence
+        id: check_files
+        uses: andstor/file-existence-action@v3
+        with:
+          files: "plugins/${{ inputs.package }}/src/js/package.json"
+
       - name: Install npm dependencies
+        if: steps.check_files.outputs.files_exists == 'true'
         run: npm ci
 
       - name: Build npm packages
+        if: steps.check_files.outputs.files_exists == 'true'
         run: npm run build -- --scope "@deephaven/js-plugin-${{ inputs.package }}"
 
       - name: Set up Python

.github/workflows/test-python-package.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: ['3.8', '3.9', '3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.gitignore

@@ -17,10 +17,21 @@ tsconfig.tsbuildinfo
 .stylelintcache
 
 # Ignore the test reports
+coverage/
 junit.xml
 
 # Allow for local overrides of docker-compose.yml. https://docs.docker.com/compose/multiple-compose-files/merge/
 docker-compose.override.yml
 
 # Ignore temporary files created during a release
 releases/
+
+# Playwright
+test-results/
+playwright-report/
+blob-report/
+playwright/.cache/
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+replay_pid*

.pre-commit-config.yaml

@@ -20,7 +20,8 @@ repos:
             deephaven-core,
             plotly,
             json-rpc,
-            matplotlib
+            matplotlib,
+            deephaven-plugin-utilities
           ]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.2.2

Dockerfile

@@ -37,6 +37,8 @@ COPY babel.config.js lerna.json nx.json tsconfig.json ./
 # This requires the Dockerfile to be built in the context of the root of the deephaven-plugins repository
 # https://stackoverflow.com/a/34300129
 COPY plugins plugins
+# delete the plotly plugin as it's deprecated
+RUN rm -rf plugins/plotly
 
 # Build the JS
 RUN npm run build
@@ -55,4 +57,4 @@ COPY --link docker/config/deephaven.prop /opt/deephaven/config/deephaven.prop
 COPY --link docker/data /data
 
 # Set the environment variable to enable the JS plugins embedded in Python
-ENV DEEPHAVEN_ENABLE_PY_JS=true
+ENV DEEPHAVEN_ENABLE_PY_JS=true

README.md

@@ -61,6 +61,14 @@ To bypass the pre-commit hook, you can commit with the `--no-verify` flag, for e
 git commit --no-verify -m "commit message"`
 ```
 
+### Running end-to-end tests
+
+We use [Playwright](https://playwright.dev/) for end-to-end tests. We test against Chrome, Firefox, and Webkit (Safari). Snapshots from E2E tests are only run against Linux so they can be validated in CI.
+
+You should be able to pass arguments to these commands as if you were running Playwright via CLI directly. For example, to test only `matplotlib.spec.ts` you could run `npm run e2e:docker -- ./tests/matplotlib.spec.ts`, or to test only `matplotlib.spec.ts` in Firefox, you could run `npm run e2e:docker -- --project firefox ./tests/matplotlib.spec.ts`. See [Playwright CLI](https://playwright.dev/docs/test-cli) for more details.
+
+It is highly recommended to use `npm run e2e:docker` (instead of `npm run e2e`) as CI also uses the same environment. You can also use `npm run e2e:update-snapshots` to regenerate snapshots in said environment.
+
 ### Running Python tests
 
 The above steps will also set up `tox` to run tests for the python plugins that support it.

cog.toml

@@ -27,9 +27,9 @@ post_package_bump_hooks = [
     # prepare the github release changelog file
     "mkdir -p ../../releases",
     "../../tools/extract_changelog.sh CHANGELOG.md > ../../releases/GITHUB_CHANGELOG-{{package}}.md",
-    # update the version number to have a .dev0 suffix (when possible, only done for python plugins)
+    # update the version number to have a `dev0` suffix
     "../../tools/update_version.sh {{package}} {{version}} --dev",
-    "git commit -m 'chore(version): update {{package}} version to {{version}}'",
+    "git commit --all --message 'chore(version): update {{package}} version to {{version}}.dev0'",
     # push the tag and the commits to main 
     "git push origin {{package}}-v{{version}}",
     "git push origin main",
@@ -76,4 +76,6 @@ plotly = { path = "plugins/plotly", public_api=false }
 plotly-express = { path = "plugins/plotly-express", public_api=false }
 table-example = { path = "plugins/table-example", public_api=false }
 ui = { path = "plugins/ui", public_api=false }
+packaging = { path = "plugins/packaging", public_api=false }
+utilities = { path = "plugins/utilities", public_api=false }
 

docker-compose.yml

@@ -7,5 +7,10 @@ services:
       pull: true
     ports:
       - '${DEEPHAVEN_PORT:-10000}:10000'
+    expose:
+      - 10000
     volumes:
       - ./docker/data/:/data
+      - ./tests/app.d:/app.d
+    environment:
+      - START_OPTS=-Xmx4g -DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler -Ddeephaven.console.type=python -Ddeephaven.application.dir=./app.d

docker/config/deephaven.prop

@@ -2,8 +2,6 @@ includefiles=dh-defaults.prop
 
 deephaven.console.type=python
 
-# Add all plugins that you want installed here
-deephaven.jsPlugins.@deephaven/js-plugin-ui=/opt/deephaven/config/plugins/plugins/ui/src/js
 
 # Anonymous authentication so we don't need to put in a password
 AuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler

docker/data/storage/notebooks/DEMO.md

@@ -61,6 +61,45 @@ def my_input():
 result = my_input()
 ```
 
+## Picker (string values)
+
+The `ui.picker` component can be used to select from a list of items. Here's a basic example for selecting from a list of string values and displaying the selected key in a text field.
+
+```python
+import deephaven.ui as ui
+from deephaven.ui import use_state
+
+
+@ui.component
+def picker():
+    value, set_value = use_state("")
+
+    # Picker for selecting values
+    pick = ui.picker(
+        "Text 1",
+        "Text 2",
+        "Text 3",
+        label="Text",
+        on_selection_change=set_value,
+        selected_key=value,
+    )
+
+    # Show current selection in a ui.text component
+    text = ui.text("Selection: " + value)
+
+    # Display picker and output in a flex column
+    return ui.flex(
+        pick,
+        text,
+        direction="column",
+        margin=10,
+        gap=10,
+    )
+
+
+result = picker()
+```
+
 ## Using Tables
 
 You can open a Deephaven Table and control it using callbacks, as well. Let\'s create a table with some data, and then create a component that allows us to filter the table, and a button group it or ungroup it.
@@ -158,7 +197,7 @@ def stock_table_input(source, default_sym="", default_exchange=""):
     return [
         ui.panel(
             # Add a callback for when user double clicks a row in the table
-            ui.table(t1).on_row_double_press(handle_row_double_press),
+            ui.table(t1, on_row_double_press=handle_row_double_press),
             title="Stock Row Press",
         ),
         ui.panel(t2, title="Stock Filtered Table"),
@@ -283,7 +322,7 @@ def table_tabs(source):
     return ui.tabs(
         ui.tab_list(
             ui.item("Unfiltered", key="Unfiltered"),
-            ui.item(ui.icon("vsGithubAlt"), "CAT", key="CAT"),
+            ui.item(ui.icon("vsGithubAlt"), ui.text("CAT"), key="CAT"),
             ui.item("DOG", key="DOG"),
         ),
         ui.tab_panels(

jest.setup.ts

@@ -24,3 +24,10 @@ Object.defineProperty(window, 'matchMedia', {
     dispatchEvent: jest.fn(),
   })),
 });
+
+/**
+ * Mock the structuredClone function to use `JSON.stringify` and `JSON.parse`
+ * This is necessary because jsdom does not support `structuredClone`.
+ * https://github.com/jsdom/jsdom/issues/3363
+ */
+global.structuredClone = jest.fn(val => JSON.parse(JSON.stringify(val)));