Stateful Selectables and SelectColumns

engine/api/src/main/java/io/deephaven/engine/table/ColumnSource.java

@@ -202,7 +202,7 @@ default <TYPE> ColumnSource<TYPE> cast(Class<? extends TYPE> clazz, @Nullable Cl
     /**
      * Most column sources will return the same value for a given row without respect to the order that the rows are
      * read. Those columns sources are considered "stateless" and should return true.
-     *
+     * <p>
      * Some column sources, however may be dependent on evaluation order. For example, a formula that updates a Map must
      * be evaluated from the first row to the last row. A column source that has the potential to depend on the order of
      * evaluation must return false.

engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java

@@ -59,11 +59,6 @@ public final boolean isRetain() {
         return false;
     }
 
-    @Override
-    public final boolean isStateless() {
-        return true;
-    }
-
     static ColumnSource<Table> getAndValidateInputColumnSource(
             @NotNull final String inputColumnName,
             @NotNull final Map<String, ? extends ColumnSource<?>> columnsOfInterest) {

engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java

@@ -105,11 +105,6 @@ public final boolean isRetain() {
         return false;
     }
 
-    @Override
-    public final boolean isStateless() {
-        return true;
-    }
-
     private static final class OutputFormulaFillContext implements Formula.FillContext {
 
         private static final Formula.FillContext INSTANCE = new OutputFormulaFillContext();

engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java

@@ -834,38 +834,4 @@ String makeGetExpression(boolean usePrev) {
             return String.format("%s.%s(k)", name, getGetterName(columnDefinition.getDataType(), usePrev));
         }
     }
-
-    /**
-     * Is this parameter immutable, and thus would contribute no state to the formula?
-     * <p>
-     * If any query scope parameter is not a primitive, String, or known immutable class; then it may be a mutable
-     * object that results in undefined results when the column is not evaluated strictly in order.
-     *
-     * @return true if this query scope parameter is immutable
-     */
-    private static boolean isImmutableType(QueryScopeParam<?> param) {
-        final Object value = param.getValue();
-        if (value == null) {
-            return true;
-        }
-        final Class<?> type = value.getClass();
-        if (type == String.class || type == BigInteger.class || type == BigDecimal.class
-                || type == Instant.class || type == ZonedDateTime.class || Table.class.isAssignableFrom(type)) {
-            return true;
-        }
-        // if it is a boxed type, then it is immutable; otherwise we don't know what to do with it
-        return TypeUtils.isBoxedType(type);
-    }
-
-    private boolean isUsedColumnStateless(String columnName) {
-        return columnSources.get(columnName).isStateless();
-    }
-
-    @Override
-    public boolean isStateless() {
-        return Arrays.stream(params).allMatch(DhFormulaColumn::isImmutableType)
-                && usedColumns.stream().allMatch(this::isUsedColumnStateless)
-                && usedColumnArrays.stream().allMatch(this::isUsedColumnStateless);
-    }
-
 }

engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java

@@ -218,11 +218,6 @@ public boolean isRetain() {
         return false;
     }
 
-    @Override
-    public boolean isStateless() {
-        return false;
-    }
-
     @Override
     public FunctionalColumn<S, D> copy() {
         return new FunctionalColumn<>(sourceName, sourceDataType, destName, destDataType, function);

engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java

@@ -197,11 +197,6 @@ public boolean isRetain() {
         return false;
     }
 
-    @Override
-    public boolean isStateless() {
-        return false;
-    }
-
     @Override
     public MultiSourceFunctionalColumn<D> copy() {
         return new MultiSourceFunctionalColumn<>(sourceNames, destName, destDataType, function);

engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java

@@ -88,11 +88,6 @@ public boolean isRetain() {
         return false;
     }
 
-    @Override
-    public boolean isStateless() {
-        return true;
-    }
-
     @Override
     public SelectColumn copy() {
         // noinspection unchecked,rawtypes

engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java

@@ -3,10 +3,7 @@
  */
 package io.deephaven.engine.table.impl.select;
 
-import io.deephaven.api.ColumnName;
-import io.deephaven.api.RawString;
-import io.deephaven.api.Selectable;
-import io.deephaven.api.Strings;
+import io.deephaven.api.*;
 import io.deephaven.api.expression.Expression;
 import io.deephaven.api.expression.Function;
 import io.deephaven.api.expression.Method;
@@ -34,6 +31,15 @@
 public interface SelectColumn extends Selectable {
 
     static SelectColumn of(Selectable selectable) {
+        if (selectable instanceof StatefulSelectable) {
+            Selectable inner = ((StatefulSelectable) selectable).innerSelectable();
+
+            SelectColumn newInner = (inner instanceof SelectColumn)
+                    ? (SelectColumn) selectable
+                    : selectable.expression().walk(new ExpressionAdapter(selectable.newColumn()));
+
+            return new StatefulSelectColumn(newInner);
+        }
         return (selectable instanceof SelectColumn)
                 ? (SelectColumn) selectable
                 : selectable.expression().walk(new ExpressionAdapter(selectable.newColumn()));
@@ -172,12 +178,6 @@ default void validateSafeForRefresh(final BaseTable<?> sourceTable) {
         // nothing to validate by default
     }
 
-    /**
-     * Returns true if this column is stateless (i.e. one row does not depend on the order of evaluation for another
-     * row).
-     */
-    boolean isStateless();
-
     /**
      * Create a copy of this SelectColumn.
      *

engine/table/src/main/java/io/deephaven/engine/table/impl/select/StatefulSelectColumn.java

@@ -0,0 +1,95 @@
+/**
+ * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+ */
+package io.deephaven.engine.table.impl.select;
+
+import io.deephaven.engine.rowset.TrackingRowSet;
+import io.deephaven.engine.table.ColumnDefinition;
+import io.deephaven.engine.table.ColumnSource;
+import io.deephaven.engine.table.WritableColumnSource;
+import io.deephaven.engine.table.impl.MatchPair;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+import java.util.Map;
+
+public class StatefulSelectColumn implements SelectColumn {
+    private final SelectColumn inner;
+
+    public StatefulSelectColumn(final SelectColumn inner) {
+        this.inner = inner;
+    }
+
+    @Override
+    public List<String> initInputs(
+            @NotNull final TrackingRowSet rowSet,
+            @NotNull final Map<String, ? extends ColumnSource<?>> columnsOfInterest) {
+        return inner.initInputs(rowSet, columnsOfInterest);
+    }
+
+    @Override
+    public List<String> initDef(
+            @NotNull final Map<String, ColumnDefinition<?>> columnDefinitionMap) {
+        return inner.initDef(columnDefinitionMap);
+    }
+
+    @Override
+    public Class<?> getReturnedType() {
+        return inner.getReturnedType();
+    }
+
+    @Override
+    public List<String> getColumns() {
+        return inner.getColumns();
+    }
+
+    @Override
+    public List<String> getColumnArrays() {
+        return inner.getColumnArrays();
+    }
+
+    @Override
+    public @NotNull ColumnSource<?> getDataView() {
+        return inner.getDataView();
+    }
+
+    @Override
+    public @NotNull ColumnSource<?> getLazyView() {
+        return inner.getLazyView();
+    }
+
+    @Override
+    public String getName() {
+        return inner.getName();
+    }
+
+    @Override
+    public MatchPair getMatchPair() {
+        return inner.getMatchPair();
+    }
+
+    @Override
+    public WritableColumnSource<?> newDestInstance(long size) {
+        return inner.newDestInstance(size);
+    }
+
+    @Override
+    public WritableColumnSource<?> newFlatDestInstance(long size) {
+        return inner.newFlatDestInstance(size);
+    }
+
+    @Override
+    public boolean isRetain() {
+        return inner.isRetain();
+    }
+
+    @Override
+    public SelectColumn copy() {
+        return new StatefulSelectColumn(inner.copy());
+    }
+
+    @Override
+    public boolean isStateless() {
+        return false;
+    }
+}

engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java

@@ -51,12 +51,6 @@ public final List<String> initDef(Map<String, ColumnDefinition<?>> columnDefinit
         return usedColumns;
     }
 
-    @Override
-    public boolean isStateless() {
-        // we can't control python
-        return false;
-    }
-
     @Override
     protected final FormulaSourceDescriptor getSourceDescriptor() {
         return new FormulaSourceDescriptor(

table-api/src/main/java/io/deephaven/api/Selectable.java

@@ -20,6 +20,16 @@
  */
 public interface Selectable {
 
+    /**
+     * Wrap a selectable to mark it as stateful.
+     *
+     * @param inner the selectable that is stateful
+     * @return the wrapped selectable
+     */
+    static StatefulSelectable stateful(Selectable inner) {
+        return new StatefulSelectable(inner);
+    }
+
     static Selectable of(ColumnName newColumn, Expression expression) {
         if (newColumn.equals(expression)) {
             return newColumn;
@@ -65,4 +75,12 @@ static List<Selectable> from(Collection<String> values) {
      * @return the expression
      */
     Expression expression();
+
+    /**
+     * @return true if this column is stateless (i.e. one row does not depend on the order of evaluation for another
+     *         row).
+     */
+    default boolean isStateless() {
+        return true;
+    }
 }

table-api/src/main/java/io/deephaven/api/StatefulSelectable.java

@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+ */
+package io.deephaven.api;
+
+import io.deephaven.api.expression.Expression;
+
+public class StatefulSelectable implements Selectable {
+    private final Selectable inner;
+
+    public StatefulSelectable(final Selectable inner) {
+        this.inner = inner;
+    }
+
+    public Selectable innerSelectable() {
+        return inner;
+    }
+
+    @Override
+    public ColumnName newColumn() {
+        return inner.newColumn();
+    }
+
+    @Override
+    public Expression expression() {
+        return inner.expression();
+    }
+
+    @Override
+    public boolean isStateless() {
+        return false;
+    }
+}