Merge remote-tracking branch 'upstream/main' into barrage_types

nbauernfeind · Dec 4, 2024 · c5000e7 · c5000e7
2 parents be9f2df + e403ffd
commit c5000e7
Show file tree

Hide file tree

Showing 85 changed files with 4,121 additions and 1,892 deletions.
diff --git a/R/rdeephaven/DESCRIPTION b/R/rdeephaven/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: rdeephaven
 Type: Package
 Title: R Client for Deephaven Core
-Version: 0.37.0
+Version: 0.38.0
 Date: 2023-05-12
 Author: Deephaven Data Labs
 Maintainer: Alex Peters <[email protected]>

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -26,7 +26,7 @@ dependencies {
         because('needed by plugin com.bmuschko.docker-remote-api')
     }
 
-    implementation('com.avast.gradle:gradle-docker-compose-plugin:0.17.10') {
+    implementation('com.avast.gradle:gradle-docker-compose-plugin:0.17.11') {
         because('needed by plugin com.avast.gradle.docker-compose')
     }
 

diff --git a/cpp-client/deephaven/CMakeLists.txt b/cpp-client/deephaven/CMakeLists.txt
@@ -8,7 +8,7 @@ endif()
 
 project(deephaven)
 
-set(deephaven_VERSION 0.37.0)
+set(deephaven_VERSION 0.38.0)
 set(CMAKE_CXX_STANDARD 17)
 
 # for CMAKE_INSTALL_{dir}

diff --git a/cpp-client/deephaven/tests/src/table_test.cc b/cpp-client/deephaven/tests/src/table_test.cc
@@ -41,8 +41,8 @@ TEST_CASE("Fetch the entire table (small)", "[client_table]") {
         "Bools = ii == 5 ? null : ((ii % 2) == 0)",
         "Strings = ii == 5 ? null : `hello ` + i",
         "DateTimes = ii == 5 ? null : '2001-03-01T12:34:56Z' + ii",
-        "LocalDates = ii == 5 ? null : parseLocalDate(`2001-3-` + (ii + 1))",
-        "LocalTimes = ii == 5 ? null : parseLocalTime(`12:34:` + (46 + ii))"
+        "LocalDates = ii == 5 ? null : '2001-03-01' + ((int)ii * 'P1D')",
+        "LocalTimes = ii == 5 ? null : '12:34:46'.plus((int)ii * 'PT1S')"
       });
   std::cout << th.Stream(true) << '\n';
 

diff --git a/cpp-client/deephaven/tests/src/ticking_test.cc b/cpp-client/deephaven/tests/src/ticking_test.cc
@@ -14,6 +14,8 @@ using deephaven::client::Client;
 using deephaven::client::TableHandle;
 using deephaven::client::utility::TableMaker;
 using deephaven::dhcore::DateTime;
+using deephaven::dhcore::LocalDate;
+using deephaven::dhcore::LocalTime;
 using deephaven::dhcore::chunk::ChunkMaker;
 using deephaven::dhcore::chunk::BooleanChunk;
 using deephaven::dhcore::chunk::DateTimeChunk;
@@ -188,6 +190,8 @@ class WaitForPopulatedTableCallback final : public CommonBase {
     auto bools = MakeReservedVector<std::optional<bool>>(target_);
     auto strings = MakeReservedVector<std::optional<std::string>>(target_);
     auto date_times = MakeReservedVector<std::optional<DateTime>>(target_);
+    auto local_dates = MakeReservedVector<std::optional<LocalDate>>(target_);
+    auto local_times = MakeReservedVector<std::optional<LocalTime>>(target_);
 
     auto date_time_start = DateTime::Parse("2001-03-01T12:34:56Z");
 
@@ -202,6 +206,8 @@ class WaitForPopulatedTableCallback final : public CommonBase {
       bools.emplace_back((i % 2) == 0);
       strings.emplace_back(fmt::format("hello {}", i));
       date_times.emplace_back(DateTime::FromNanos(date_time_start.Nanos() + i));
+      local_dates.emplace_back(LocalDate::Of(2001, 3, i + 1));
+      local_times.emplace_back(LocalTime::Of(12, 34, 46 + i));
     }
 
     if (target_ == 0) {
@@ -220,6 +226,8 @@ class WaitForPopulatedTableCallback final : public CommonBase {
     bools[t2] = {};
     strings[t2] = {};
     date_times[t2] = {};
+    local_dates[t2] = {};
+    local_times[t2] = {};
 
     CompareColumn(*current, "Chars", chars);
     CompareColumn(*current, "Bytes", int8s);
@@ -231,6 +239,8 @@ class WaitForPopulatedTableCallback final : public CommonBase {
     CompareColumn(*current, "Bools", bools);
     CompareColumn(*current, "Strings", strings);
     CompareColumn(*current, "DateTimes", date_times);
+    CompareColumn(*current, "LocalDates", local_dates);
+    CompareColumn(*current, "LocalTimes", local_times);
 
     NotifyDone();
   }
@@ -255,7 +265,9 @@ TEST_CASE("Ticking Table: all the data is eventually present", "[ticking]") {
           "Doubles = II == 5 ? null : (double)II",
           "Bools = II == 5 ? null : ((II % 2) == 0)",
           "Strings = II == 5 ? null : (`hello ` + II)",
-          "DateTimes = II == 5 ? null : ('2001-03-01T12:34:56Z' + II)"
+          "DateTimes = II == 5 ? null : ('2001-03-01T12:34:56Z' + II)",
+          "LocalDates = ii == 5 ? null : '2001-03-01' + ((int)II * 'P1D')",
+          "LocalTimes = ii == 5 ? null : '12:34:46'.plus((int)II * 'PT1S')"
       })
       .LastBy("II")
       .Sort(SortPair::Ascending("II"));

diff --git a/csharp/ExcelAddInInstaller/ExcelAddInInstaller.aip b/csharp/ExcelAddInInstaller/ExcelAddInInstaller.aip
@@ -8,6 +8,9 @@
     <ROW Property="AI_ThemeStyle" Value="aero" MultiBuildValue="DefaultBuild:style6blue" MsiKey="AI_ThemeStyle"/>
     <ROW Property="ALLUSERS" Value="1" MultiBuildValue="DefaultBuild:"/>
     <ROW Property="ARPCOMMENTS" Value="This installer database contains the logic and data required to install [|ProductName]." ValueLocId="*"/>
+    <ROW Property="ARPPRODUCTICON" Value="Icon.exe" Type="8"/>
+    <ROW Property="BannerBitmap" Value="banner.jpg" MultiBuildValue="DefaultBuild:Banner.jpg" Type="1" MsiKey="BannerBitmap"/>
+    <ROW Property="DialogBitmap" Value="dialog.jpg" MultiBuildValue="DefaultBuild:Dialog.jpg" Type="1" MsiKey="DialogBitmap"/>
     <ROW Property="Manufacturer" Value="Deephaven Data Labs LLC"/>
     <ROW Property="ProductCode" Value="1033:{5C1E1B63-CC23-4F4D-8564-A843AA5D7413} " Type="16"/>
     <ROW Property="ProductLanguage" Value="1033"/>
@@ -140,6 +143,8 @@
     <ROW Action="AI_VerifyPrereq" Description="Verifying prerequisites" DescriptionLocId="ActionText.Description.AI_VerifyPrereq" Template="[1] was not installed correctly." TemplateLocId="ActionText.Template.AI_VerifyPrereq"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiBinaryComponent">
+    <ROW Name="Banner.jpg" SourcePath="resources\Banner.jpg"/>
+    <ROW Name="Dialog.jpg" SourcePath="resources\Dialog.jpg"/>
     <ROW Name="DotNetMethodCaller.dll" SourcePath="&lt;AI_CUSTACTS&gt;DotNetMethodCaller.dll"/>
     <ROW Name="DotNetMethodCaller.dll_1" SourcePath="&lt;AI_CUSTACTS64&gt;DotNetMethodCaller.dll"/>
     <ROW Name="Prereq.dll" SourcePath="&lt;AI_CUSTACTS&gt;Prereq.dll"/>
@@ -263,6 +268,9 @@
     <ROW Feature_="MainFeature" Component_="zlib1.dll"/>
     <ROW Feature_="MainFeature" Component_="zstd.dll"/>
   </COMPONENT>
+  <COMPONENT cid="caphyon.advinst.msicomp.MsiIconsComponent">
+    <ROW Name="Icon.exe" SourcePath="resources\Icon.ico" Index="0"/>
+  </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiInstExSeqComponent">
     <ROW Action="AI_DOWNGRADE" Condition="AI_NEWERPRODUCTFOUND AND (UILevel &lt;&gt; 5)" Sequence="210"/>
     <ROW Action="AI_RESTORE_LOCATION" Condition="APPDIR=&quot;&quot;" Sequence="749"/>

diff --git a/csharp/ExcelAddInInstaller/resources/Banner.jpg b/csharp/ExcelAddInInstaller/resources/Banner.jpg
diff --git a/csharp/ExcelAddInInstaller/resources/Dialog.jpg b/csharp/ExcelAddInInstaller/resources/Dialog.jpg
diff --git a/csharp/ExcelAddInInstaller/resources/Icon.ico b/csharp/ExcelAddInInstaller/resources/Icon.ico
diff --git a/docker/registry/fedora/gradle.properties b/docker/registry/fedora/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=fedora:39
-deephaven.registry.imageId=fedora@sha256:f23412a1ad7c430fc5ed9c029b15715aed3d50e6322902a066869310cddaf915
+deephaven.registry.imageId=fedora@sha256:d63d63fe593749a5e8dbc8152427d40bbe0ece53d884e00e5f3b44859efa5077
diff --git a/docker/registry/go/gradle.properties b/docker/registry/go/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=golang:1
-deephaven.registry.imageId=golang@sha256:ad5c126b5cf501a8caef751a243bb717ec204ab1aa56dc41dc11be089fafcb4f
+deephaven.registry.imageId=golang@sha256:73f06be4578c9987ce560087e2e2ea6485fb605e3910542cadd8fa09fc5f3e31
diff --git a/docker/registry/manylinux2014_x86_64/gradle.properties b/docker/registry/manylinux2014_x86_64/gradle.properties
@@ -1,4 +1,4 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=quay.io/pypa/manylinux2014_x86_64:latest
-deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:1b6743f7785bd1630d5fc985c898b1ac46d64ad7a39173bbfc8ad2ba315a70cf
+deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:24858373a047b97fd3a8fe2ee28709479a828fa0ed88719158b728947eb53270
 deephaven.registry.platform=linux/amd64
diff --git a/docker/registry/minio/gradle.properties b/docker/registry/minio/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=minio/minio:latest
-deephaven.registry.imageId=minio/minio@sha256:9535594ad4122b7a78c6632788a989b96d9199b483d3bd71a5ceae73a922cdfa
+deephaven.registry.imageId=minio/minio@sha256:ac591851803a79aee64bc37f66d77c56b0a4b6e12d9e5356380f4105510f2332
diff --git a/docker/registry/protoc-base/gradle.properties b/docker/registry/protoc-base/gradle.properties
@@ -1,5 +1,5 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/protoc-base:latest
-deephaven.registry.imageId=ghcr.io/deephaven/protoc-base@sha256:bbefcc311f3bd93da25d102173dd83a1b5244743186692d5251db71cb7186567
+deephaven.registry.imageId=ghcr.io/deephaven/protoc-base@sha256:3cb04f768cfdb66b47154568a23772d42dcc55076c7cbb94ae950309d350a82a
 # TODO(deephaven-base-images#54): arm64 native image for cpp-client-base
 deephaven.registry.platform=linux/amd64
diff --git a/docker/registry/python/gradle.properties b/docker/registry/python/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=python:3.10
-deephaven.registry.imageId=python@sha256:fd0fa50d997eb56ce560c6e5ca6a1f5cf8fdff87572a16ac07fb1f5ca01eb608
+deephaven.registry.imageId=python@sha256:941b0bfddbf17d809fd1f457acbf55dfca014e3e0e3d592b1c9070491681bc02
diff --git a/docker/registry/server-base/gradle.properties b/docker/registry/server-base/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/server-base:edge
-deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:07cc7f322d57cfe86f174b88d6315ce25bac1b798c14ddb9b833b918810f69ef
+deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:6885027cbb05b7739a78be26393b1e22085ee1667640c0d9d05a95f8f561d819
diff --git a/docker/registry/slim-base/gradle.properties b/docker/registry/slim-base/gradle.properties
@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/server-slim-base:edge
-deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:a01198a4af6bab387a337174fb2babe599e2bd0225a42ff9be31f3052d70e022
+deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:941e783bde091416f4b0359cc2a9df893f8340ada9bf4681f84875a91dfcebd1
diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt
@@ -1,5 +1,5 @@
-adbc-driver-manager==1.2.0
-adbc-driver-postgresql==1.2.0
+adbc-driver-manager==1.3.0
+adbc-driver-postgresql==1.3.0
 connectorx==0.4.0; platform.machine == 'x86_64'
 deephaven-plugin==0.6.0
 docstring_parser==0.16

diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt
@@ -1,5 +1,5 @@
-adbc-driver-manager==1.2.0
-adbc-driver-postgresql==1.2.0
+adbc-driver-manager==1.3.0
+adbc-driver-postgresql==1.3.0
 connectorx==0.4.0; platform.machine == 'x86_64'
 deephaven-plugin==0.6.0
 docstring_parser==0.16

diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java
@@ -264,18 +264,17 @@ public void testCollidingCompile() throws Exception {
 
     @Test
     public void testMultiCompileWithFailure() throws ExecutionException, InterruptedException {
-        final String goodProgram = String.join(
-                "\n",
+        final String goodProgram = String.join("\n",
                 "public class GoodTest {",
                 "   public static void main (String [] args) {",
                 "   }",
                 "}");
-        final String badProgram = String.join(
-                "\n",
-                "public class BadTest {",
-                "   public static void main (String [] args) {",
-                "   }",
-                "}}");
+        final String badProgram = String.join("\n",
+                "public class Formula {",
+                "    public Formula() {",
+                "        S.badCall(0);",
+                "    }",
+                "}");
 
         QueryCompilerRequest[] requests = new QueryCompilerRequest[] {
                 QueryCompilerRequest.builder()
@@ -299,15 +298,53 @@ public void testMultiCompileWithFailure() throws ExecutionException, Interrupted
                         CompletionStageFuture.make(),
                 };
 
-        try {
-            ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers);
-            // noinspection DataFlowIssue
-            throw Assert.statementNeverExecuted();
-        } catch (Exception ignored) {
-        }
+        ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers);
 
         Assert.eqTrue(resolvers[0].getFuture().isDone(), "resolvers[0].getFuture().isDone()");
         Assert.eqTrue(resolvers[1].getFuture().isDone(), "resolvers[0].getFuture().isDone()");
         Assert.neqNull(resolvers[1].getFuture().get(), "resolvers[1].getFuture().get()");
     }
+
+    @Test
+    public void testMultiCompileWithFailureSecond() throws ExecutionException, InterruptedException {
+        final String badProgram = String.join("\n",
+                "public class Formula {",
+                "    public Formula() {",
+                "        S.badCall(0);",
+                "    }",
+                "}");
+        final String goodProgram = String.join("\n",
+                "public class Formula {",
+                "   public static void main (String [] args) {",
+                "   }",
+                "}");
+
+        QueryCompilerRequest[] requests = new QueryCompilerRequest[] {
+                QueryCompilerRequest.builder()
+                        .description("Test Good Compile")
+                        .className("Formula")
+                        .classBody(goodProgram)
+                        .packageNameRoot("com.deephaven.test")
+                        .build(),
+                QueryCompilerRequest.builder()
+                        .description("Test Bad Compile")
+                        .className("Formula")
+                        .classBody(badProgram)
+                        .packageNameRoot("com.deephaven.test")
+                        .build(),
+        };
+
+        // noinspection unchecked
+        CompletionStageFuture.Resolver<Class<?>>[] resolvers =
+                (CompletionStageFuture.Resolver<Class<?>>[]) new CompletionStageFuture.Resolver[] {
+                        CompletionStageFuture.make(),
+                        CompletionStageFuture.make(),
+                };
+
+        ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers);
+
+        Assert.eqTrue(resolvers[1].getFuture().isDone(), "resolvers[0].getFuture().isDone()");
+        Assert.eqTrue(resolvers[0].getFuture().isDone(), "resolvers[0].getFuture().isDone()");
+        Assert.neqNull(resolvers[0].getFuture().get(), "resolvers[1].getFuture().get()");
+    }
 }
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/AsOfJoinHelper.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/AsOfJoinHelper.java
@@ -1340,8 +1340,10 @@ public void onUpdate(TableUpdate upstream) {
                     @Override
                     protected void destroy() {
                         super.destroy();
-                        leftStampKeys.close();
-                        leftStampValues.close();
+                        getUpdateGraph().runWhenIdle(() -> {
+                            leftStampKeys.close();
+                            leftStampValues.close();
+                        });
                     }
                 });
 
@@ -1522,8 +1524,10 @@ public void onUpdate(TableUpdate upstream) {
                                 @Override
                                 protected void destroy() {
                                     super.destroy();
-                                    compactedRightStampKeys.close();
-                                    compactedRightStampValues.close();
+                                    getUpdateGraph().runWhenIdle(() -> {
+                                        compactedRightStampKeys.close();
+                                        compactedRightStampValues.close();
+                                    });
                                 }
                             });
 

diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java
@@ -113,6 +113,21 @@ public SafeCloseable beginOperation(@NotNull final QueryTable parent) {
     void initializeFilters(@NotNull QueryTable parent) {
         final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = QueryCompilerRequestProcessor.batch();
         Arrays.stream(whereFilters).forEach(filter -> filter.init(parent.getDefinition(), compilationProcessor));
+
+        final List<WhereFilter> disallowedRowVariables =
+                Arrays.stream(whereFilters).filter(WhereFilter::hasVirtualRowVariables).collect(Collectors.toList());
+        if (!disallowedRowVariables.isEmpty()) {
+            throw new UncheckedTableException(
+                    "wouldMatch filters cannot use virtual row variables (i, ii, and k): " + disallowedRowVariables);
+        }
+
+        final List<WhereFilter> disallowedColumnVectors =
+                Arrays.stream(whereFilters).filter(wf -> !wf.getColumnArrays().isEmpty()).collect(Collectors.toList());
+        if (!disallowedColumnVectors.isEmpty()) {
+            throw new UncheckedTableException(
+                    "wouldMatch filters cannot use column Vectors (_ syntax): " + disallowedColumnVectors);
+        }
+
         compilationProcessor.compile();
     }
 
@@ -517,13 +532,14 @@ private RowSet update(RowSet added, RowSet removed, RowSet modified,
 
             try (final SafeCloseableList toClose = new SafeCloseableList()) {
                 // Filter and add addeds
-                final WritableRowSet filteredAdded = toClose.add(filter.filter(added, source, table, false));
+                final WritableRowSet filteredAdded = toClose.add(filter.filter(added, table.getRowSet(), table, false));
                 RowSet keysToRemove = EMPTY_INDEX;
 
                 // If we were affected, recompute mods and re-add the ones that pass.
                 if (affected) {
                     downstreamModified.setAll(name);
-                    final RowSet filteredModified = toClose.add(filter.filter(modified, source, table, false));
+                    final RowSet filteredModified =
+                            toClose.add(filter.filter(modified, table.getRowSet(), table, false));
 
                     // Now apply the additions and remove any non-matching modifieds
                     filteredAdded.insert(filteredModified);

diff --git a/...rc/main/java/io/deephaven/engine/table/impl/asofjoin/BucketedChunkedAjMergedListener.java b/...rc/main/java/io/deephaven/engine/table/impl/asofjoin/BucketedChunkedAjMergedListener.java
@@ -791,7 +791,9 @@ private RowSet indexFromBuilder(int slotIndex) {
     @Override
     protected void destroy() {
         super.destroy();
-        leftSsaFactory.close();
-        rightSsaFactory.close();
+        getUpdateGraph().runWhenIdle(() -> {
+            leftSsaFactory.close();
+            rightSsaFactory.close();
+        });
     }
 }
diff --git a/...ne/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/...ne/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java
@@ -311,6 +311,11 @@ public boolean hasConstantArrayAccess() {
         return getFormulaShiftColPair() != null;
     }
 
+    @Override
+    public boolean hasVirtualRowVariables() {
+        return usesI || usesII || usesK;
+    }
+
     /**
      * @return a Pair object, consisting of formula string and shift to column MatchPairs, if the filter formula or
      *         expression has Array Access that conforms to "i +/- &lt;constant&gt;" or "ii +/- &lt;constant&gt;". If