Add column source reinterpretation for UpdateBy Delta operator. (#5043)

deephaven · Jan 16, 2024 · 63bfd97 · 63bfd97
1 parent 51546a5
commit 63bfd97
Show file tree

Hide file tree

Showing 8 changed files with 74 additions and 13 deletions.
diff --git a/.../table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/ByteDeltaOperator.java b/.../table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/ByteDeltaOperator.java
@@ -14,6 +14,7 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.MatchPair;
+import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.updateby.UpdateByOperator;
 import io.deephaven.engine.table.impl.updateby.internal.BaseByteUpdateByOperator;
 import io.deephaven.engine.table.impl.util.RowRedirection;
@@ -74,7 +75,7 @@ public ByteDeltaOperator(@NotNull final MatchPair pair,
     ) {
         super(pair, new String[] { pair.rightColumn }, rowRedirection);
         this.control = control;
-        this.inputSource = inputSource;
+        this.inputSource = ReinterpretUtils.maybeConvertToPrimitive(inputSource);
         // region constructor
         // endregion constructor
     }

diff --git a/.../table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/CharDeltaOperator.java b/.../table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/CharDeltaOperator.java
@@ -9,6 +9,7 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.MatchPair;
+import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.updateby.UpdateByOperator;
 import io.deephaven.engine.table.impl.updateby.internal.BaseCharUpdateByOperator;
 import io.deephaven.engine.table.impl.util.RowRedirection;
@@ -69,7 +70,7 @@ public CharDeltaOperator(@NotNull final MatchPair pair,
     ) {
         super(pair, new String[] { pair.rightColumn }, rowRedirection);
         this.control = control;
-        this.inputSource = inputSource;
+        this.inputSource = ReinterpretUtils.maybeConvertToPrimitive(inputSource);
         // region constructor
         // endregion constructor
     }

diff --git a/...able/src/main/java/io/deephaven/engine/table/impl/updateby/delta/DoubleDeltaOperator.java b/...able/src/main/java/io/deephaven/engine/table/impl/updateby/delta/DoubleDeltaOperator.java
@@ -14,6 +14,7 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.MatchPair;
+import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.updateby.UpdateByOperator;
 import io.deephaven.engine.table.impl.updateby.internal.BaseDoubleUpdateByOperator;
 import io.deephaven.engine.table.impl.util.RowRedirection;
@@ -74,7 +75,7 @@ public DoubleDeltaOperator(@NotNull final MatchPair pair,
     ) {
         super(pair, new String[] { pair.rightColumn }, rowRedirection);
         this.control = control;
-        this.inputSource = inputSource;
+        this.inputSource = ReinterpretUtils.maybeConvertToPrimitive(inputSource);
         // region constructor
         // endregion constructor
     }

diff --git a/...table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/FloatDeltaOperator.java b/...table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/FloatDeltaOperator.java
@@ -14,6 +14,7 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.MatchPair;
+import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.updateby.UpdateByOperator;
 import io.deephaven.engine.table.impl.updateby.internal.BaseFloatUpdateByOperator;
 import io.deephaven.engine.table.impl.util.RowRedirection;
@@ -74,7 +75,7 @@ public FloatDeltaOperator(@NotNull final MatchPair pair,
     ) {
         super(pair, new String[] { pair.rightColumn }, rowRedirection);
         this.control = control;
-        this.inputSource = inputSource;
+        this.inputSource = ReinterpretUtils.maybeConvertToPrimitive(inputSource);
         // region constructor
         // endregion constructor
     }

diff --git a/...e/table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/IntDeltaOperator.java b/...e/table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/IntDeltaOperator.java
@@ -14,6 +14,7 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.MatchPair;
+import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.updateby.UpdateByOperator;
 import io.deephaven.engine.table.impl.updateby.internal.BaseIntUpdateByOperator;
 import io.deephaven.engine.table.impl.util.RowRedirection;
@@ -74,7 +75,7 @@ public IntDeltaOperator(@NotNull final MatchPair pair,
     ) {
         super(pair, new String[] { pair.rightColumn }, rowRedirection);
         this.control = control;
-        this.inputSource = inputSource;
+        this.inputSource = ReinterpretUtils.maybeConvertToPrimitive(inputSource);
         // region constructor
         // endregion constructor
     }

diff --git a/.../table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/LongDeltaOperator.java b/.../table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/LongDeltaOperator.java
@@ -14,6 +14,7 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.MatchPair;
+import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.updateby.UpdateByOperator;
 import io.deephaven.engine.table.impl.updateby.internal.BaseLongUpdateByOperator;
 import io.deephaven.engine.table.impl.util.RowRedirection;
@@ -74,7 +75,7 @@ public LongDeltaOperator(@NotNull final MatchPair pair,
     ) {
         super(pair, new String[] { pair.rightColumn }, rowRedirection);
         this.control = control;
-        this.inputSource = inputSource;
+        this.inputSource = ReinterpretUtils.maybeConvertToPrimitive(inputSource);
         // region constructor
         // endregion constructor
     }

diff --git a/...table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/ShortDeltaOperator.java b/...table/src/main/java/io/deephaven/engine/table/impl/updateby/delta/ShortDeltaOperator.java
@@ -14,6 +14,7 @@
 import io.deephaven.engine.rowset.RowSet;
 import io.deephaven.engine.table.ColumnSource;
 import io.deephaven.engine.table.impl.MatchPair;
+import io.deephaven.engine.table.impl.sources.ReinterpretUtils;
 import io.deephaven.engine.table.impl.updateby.UpdateByOperator;
 import io.deephaven.engine.table.impl.updateby.internal.BaseShortUpdateByOperator;
 import io.deephaven.engine.table.impl.util.RowRedirection;
@@ -74,7 +75,7 @@ public ShortDeltaOperator(@NotNull final MatchPair pair,
     ) {
         super(pair, new String[] { pair.rightColumn }, rowRedirection);
         this.control = control;
-        this.inputSource = inputSource;
+        this.inputSource = ReinterpretUtils.maybeConvertToPrimitive(inputSource);
         // region constructor
         // endregion constructor
     }

diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestDelta.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/updateby/TestDelta.java
@@ -3,20 +3,19 @@
 import io.deephaven.api.updateby.DeltaControl;
 import io.deephaven.api.updateby.NullBehavior;
 import io.deephaven.api.updateby.UpdateByOperation;
+import io.deephaven.base.verify.Assert;
 import io.deephaven.engine.context.ExecutionContext;
-import io.deephaven.engine.table.PartitionedTable;
-import io.deephaven.engine.table.Table;
+import io.deephaven.engine.table.*;
 import io.deephaven.engine.table.impl.DataAccessHelpers;
 import io.deephaven.engine.table.impl.QueryTable;
-import io.deephaven.engine.testutil.ControlledUpdateGraph;
-import io.deephaven.engine.testutil.EvalNugget;
-import io.deephaven.engine.testutil.GenerateTableUpdates;
-import io.deephaven.engine.testutil.TstUtils;
+import io.deephaven.engine.testutil.*;
 import io.deephaven.engine.testutil.generator.CharGenerator;
 import io.deephaven.engine.testutil.generator.SortedInstantGenerator;
 import io.deephaven.engine.testutil.generator.TestDataGenerator;
+import io.deephaven.engine.util.TableTools;
 import io.deephaven.test.types.OutOfBandTest;
 import io.deephaven.time.DateTimeUtils;
+import io.deephaven.util.SafeCloseable;
 import org.jetbrains.annotations.NotNull;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -25,10 +24,13 @@
 import java.math.BigInteger;
 import java.time.Instant;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Random;
 
 import static io.deephaven.engine.testutil.GenerateTableUpdates.generateAppends;
+import static io.deephaven.engine.testutil.TstUtils.getTable;
+import static io.deephaven.engine.testutil.TstUtils.initColumnInfos;
 import static io.deephaven.engine.testutil.testcase.RefreshingTableTestCase.simulateShiftAwareStep;
 import static io.deephaven.engine.util.TableTools.intCol;
 import static io.deephaven.util.QueryConstants.*;
@@ -312,6 +314,58 @@ public void testBucketedGeneralTicking() {
         }
     }
 
+    @Test
+    public void testDHC5040() {
+        // ensure that Instant columns are handled correctly (interpreted to longs internally).
+        try (final SafeCloseable ignored = ExecutionContext.newBuilder()
+                .newQueryLibrary("DEFAULT")
+                .captureQueryCompiler()
+                .captureQueryScope()
+                .build().open();) {
+            ExecutionContext.getContext().getQueryLibrary().importStatic(TableTools.class);
+
+            final Random billy = new Random(0xB177B177);
+            int size = 1;
+
+            final ColumnInfo<?, ?>[] infos = initColumnInfos(
+                    new String[] {"Timestamp"},
+                    new SortedInstantGenerator(DateTimeUtils.parseInstant("2015-09-11T09:30:00 NY"),
+                            DateTimeUtils.parseInstant("2015-09-11T10:00:00 NY")));
+
+            final QueryTable timeTable = getTable(false, size, billy, infos);
+            timeTable.setRefreshing(true);
+
+            final Table st = timeTable
+                    .update("Minutes=(long)(epochNanos(Timestamp)/60_000_000_000)",
+                            "TimestampTable=newTable(longCol(\"Minutes\", Minutes), instantCol(\"Timestamp2\", Timestamp)).updateView(\"Timestamp2=Timestamp2+1\")")
+                    .dropColumns("Timestamp");
+
+            final TableDefinition td = TableDefinition.of(
+                    ColumnDefinition.ofTime("Timestamp2"),
+                    ColumnDefinition.ofLong("Minutes"));
+
+            final PartitionedTable pt = PartitionedTableFactory.of(
+                    st,
+                    Collections.singleton("Minutes"),
+                    false,
+                    "TimestampTable",
+                    td,
+                    true);
+
+            final Table mt = pt.merge();
+
+            final Table ut = mt.updateBy(UpdateByOperation.Delta("NanosDelta=Timestamp2"), "Minutes");
+
+            for (int ii = 0; ii < DYNAMIC_UPDATE_STEPS; ii++) {
+                ExecutionContext.getContext().getUpdateGraph().<ControlledUpdateGraph>cast().runWithinUnitTestCycle(
+                        () -> generateAppends(1, billy, timeTable, infos));
+                Assert.eqFalse(st.isFailed(), "st.isFailed()");
+                Assert.eqFalse(ut.isFailed(), "ut.isFailed()");
+                Assert.eq(st.size(), "st.size()", ut.size(), "ut.size()");
+            }
+        }
+    }
+
     /*
      * Ideas for specialized tests: 1) Remove first index 2) Removed everything, add some back 3) Make sandwiches
      */