Merge remote-tracking branch 'upstream/main' into barrage_types

deephaven · Dec 30, 2024 · b948edf · b948edf
2 parents 8febaa0 + 0ae8509
commit b948edf
Show file tree

Hide file tree

Showing 118 changed files with 5,336 additions and 3,130 deletions.
diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -9,6 +9,8 @@ on:
 jobs:
   linkChecker:
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
     if: ${{ github.repository_owner == 'deephaven' }}
     steps:
       - uses: actions/checkout@v4

diff --git a/README.md b/README.md
@@ -31,7 +31,7 @@ This README is intended to provide a high-level overview of the installation and
 | Go            | No                 | Yes                |
 | R             | No                 | Yes                |
 
-Deephaven's client APIs use [gRPC](https://grpc.io/), [protobuf](https://github.com/deephaven/deephaven-core/tree/main/proto/proto-backplane-grpc/src/main/proto/deephaven/proto), [Apache Arrow Flight](https://arrow.apache.org/docs/format/Flight.html), and [Barrage](https://github.com/deephaven/barrage) to handle ticking data. Users who wish to build their own client APIs can use these tools to do so.
+Deephaven's client APIs use [gRPC](https://grpc.io/), [protobuf](https://github.com/deephaven/deephaven-core/tree/main/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto), [Apache Arrow Flight](https://arrow.apache.org/docs/format/Flight.html), and [Barrage](https://github.com/deephaven/barrage) to handle ticking data. Users who wish to build their own client APIs can use these tools to do so.
 
 The following list contains documentation links for installation instructions and more:
 

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -26,7 +26,7 @@ dependencies {
         because('needed by plugin com.bmuschko.docker-remote-api')
     }
 
-    implementation('com.avast.gradle:gradle-docker-compose-plugin:0.17.11') {
+    implementation('com.avast.gradle:gradle-docker-compose-plugin:0.17.12') {
         because('needed by plugin com.avast.gradle.docker-compose')
     }
 

diff --git a/docker/registry/selenium/gradle.properties b/docker/registry/selenium/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
-deephaven.registry.imageName=selenium/standalone-firefox:4.26.0-20241101
-deephaven.registry.imageId=selenium/standalone-firefox@sha256:8625701dabb242a8092e53fae04e2eae4eb1b027ed3d2d1bef89e1fc448f9ee8
+deephaven.registry.imageName=selenium/standalone-firefox:4.27.0
+deephaven.registry.imageId=selenium/standalone-firefox@sha256:287cd73a59d241b7264fe5fab7dbd31f92d56b8ec81ba74f53ecc4997f218dff
diff --git a/docker/registry/server-base/gradle.properties b/docker/registry/server-base/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/server-base:edge
-deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:6885027cbb05b7739a78be26393b1e22085ee1667640c0d9d05a95f8f561d819
+deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:57096a26bb621a4eaf630ac555a4c08836820e029aa90c266765142c524419a4
diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt
@@ -6,15 +6,15 @@ docstring_parser==0.16
 importlib_resources==6.4.5
 java-utilities==0.3.0
 jedi==0.19.1
-jpy==0.19.0
+jpy==1.0.0
 llvmlite==0.43.0
 numba==0.60.0
 numpy==2.0.2
 pandas==2.2.3
 parso==0.8.4
-pyarrow==18.0.0
+pyarrow==18.1.0
 python-dateutil==2.9.0.post0
 pytz==2024.2
-six==1.16.0
+six==1.17.0
 typing_extensions==4.12.2
 tzdata==2024.2
diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt
@@ -6,15 +6,15 @@ docstring_parser==0.16
 importlib_resources==6.4.5
 java-utilities==0.3.0
 jedi==0.19.1
-jpy==0.19.0
+jpy==1.0.0
 llvmlite==0.43.0
 numba==0.60.0
 numpy==2.0.2
 pandas==2.2.3
 parso==0.8.4
-pyarrow==18.0.0
+pyarrow==18.1.0
 python-dateutil==2.9.0.post0
 pytz==2024.2
-six==1.16.0
+six==1.17.0
 typing_extensions==4.12.2
 tzdata==2024.2
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java
@@ -92,6 +92,7 @@
 import io.deephaven.engine.table.impl.by.ssmminmax.SsmChunkedMinMaxOperator;
 import io.deephaven.engine.table.impl.by.ssmpercentile.SsmChunkedPercentileOperator;
 import io.deephaven.engine.table.impl.select.SelectColumn;
+import io.deephaven.engine.table.impl.select.WhereFilter;
 import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.ssms.SegmentedSortedMultiSet;
 import io.deephaven.engine.table.impl.util.freezeby.FreezeByCountOperator;
@@ -107,13 +108,7 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -693,6 +688,56 @@ public void visit(@NotNull final Count count) {
             addNoInputOperator(new CountAggregationOperator(count.column().name()));
         }
 
+        @Override
+        public void visit(@NotNull final CountWhere countWhere) {
+            final WhereFilter[] whereFilters = WhereFilter.fromInternal(countWhere.filter());
+
+            final Map<String, RecordingInternalOperator> inputColumnRecorderMap = new HashMap<>();
+            final List<RecordingInternalOperator> recorderList = new ArrayList<>();
+            final List<RecordingInternalOperator[]> filterRecorderList = new ArrayList<>();
+
+            // Verify all the columns in the where filters are present in the table and valid for use.
+            for (final WhereFilter whereFilter : whereFilters) {
+                whereFilter.init(table.getDefinition());
+                if (whereFilter.isRefreshing()) {
+                    throw new UnsupportedOperationException("AggCountWhere does not support refreshing filters");
+                }
+
+                // Compute which recording operators this filter will use.
+                final List<String> inputColumnNames = whereFilter.getColumns();
+                final int inputColumnCount = whereFilter.getColumns().size();
+                final RecordingInternalOperator[] recorders = new RecordingInternalOperator[inputColumnCount];
+                for (int ii = 0; ii < inputColumnCount; ++ii) {
+                    final String inputColumnName = inputColumnNames.get(ii);
+                    final RecordingInternalOperator recorder =
+                            inputColumnRecorderMap.computeIfAbsent(inputColumnName, k -> {
+                                // Create a recording operator for the column and add it to the list of operators.
+                                final ColumnSource<?> inputSource = table.getColumnSource(inputColumnName);
+                                final RecordingInternalOperator newRecorder =
+                                        new RecordingInternalOperator(inputColumnName, inputSource);
+                                recorderList.add(newRecorder);
+                                return newRecorder;
+                            });
+                    recorders[ii] = recorder;
+                }
+                filterRecorderList.add(recorders);
+            }
+
+            final RecordingInternalOperator[] recorders = recorderList.toArray(RecordingInternalOperator[]::new);
+            final RecordingInternalOperator[][] filterRecorders =
+                    filterRecorderList.toArray(RecordingInternalOperator[][]::new);
+            final String[] inputColumnNames =
+                    inputColumnRecorderMap.keySet().toArray(ArrayTypeUtils.EMPTY_STRING_ARRAY);
+
+            // Add the recording operators, making them dependent on all input columns so they all are populated if any
+            // are modified
+            for (final RecordingInternalOperator recorder : recorders) {
+                addOperator(recorder, recorder.getInputColumnSource(), inputColumnNames);
+            }
+            addOperator(new CountWhereOperator(countWhere.column().name(), whereFilters, recorders, filterRecorders),
+                    null, inputColumnNames);
+        }
+
         @Override
         public void visit(@NotNull final FirstRowKey firstRowKey) {
             addFirstOrLastOperators(true, firstRowKey.column().name());
@@ -1004,6 +1049,11 @@ public void visit(@NotNull final Count count) {
             addNoInputOperator(new CountAggregationOperator(count.column().name()));
         }
 
+        @Override
+        public void visit(@NotNull final CountWhere countWhere) {
+            addNoInputOperator(new CountAggregationOperator(countWhere.column().name()));
+        }
+
         @Override
         public void visit(@NotNull final NullColumns nullColumns) {
             transformers.add(new NullColumnAggregationTransformer(nullColumns.resultColumns()));
@@ -1149,6 +1199,13 @@ public void visit(@NotNull final Count count) {
             addOperator(makeSumOperator(resultSource.getType(), resultName, false), resultSource, resultName);
         }
 
+        @Override
+        public void visit(@NotNull final CountWhere countWhere) {
+            final String resultName = countWhere.column().name();
+            final ColumnSource<?> resultSource = table.getColumnSource(resultName);
+            addOperator(makeSumOperator(resultSource.getType(), resultName, false), resultSource, resultName);
+        }
+
         @Override
         public void visit(@NotNull final NullColumns nullColumns) {
             transformers.add(new NullColumnAggregationTransformer(nullColumns.resultColumns()));