From 9744fc1c206d5df530b72b2dee7999ad670f840c Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Thu, 15 Aug 2024 10:10:11 -0600 Subject: [PATCH] All of our review comments from yesterday. --- .../impl/QueryCompilerRequestProcessor.java | 94 ++++++++++++++----- .../engine/table/impl/QueryTable.java | 33 ++----- .../table/impl/lang/QueryLanguageParser.java | 29 +++++- .../impl/select/AbstractConditionFilter.java | 6 +- .../impl/select/AbstractFormulaColumn.java | 12 ++- .../table/impl/select/DhFormulaColumn.java | 5 +- .../analyzers/SelectAndViewAnalyzer.java | 87 +++++++++-------- .../impl/select/codegen/FormulaAnalyzer.java | 26 ++--- .../BoxedBooleanArrayExpansionKernel.java | 2 +- 9 files changed, 184 insertions(+), 110 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java index ae321271be4..4aa259851a8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -8,7 +8,10 @@ import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.context.QueryScope; +import io.deephaven.engine.rowset.TrackingWritableRowSet; +import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.util.MultiException; import io.deephaven.util.SafeCloseable; @@ -18,26 +21,29 @@ import org.jetbrains.annotations.VisibleForTesting; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -public interface QueryCompilerRequestProcessor { +public abstract class QueryCompilerRequestProcessor { /** * @return An immediate QueryCompilerRequestProcessor */ - static QueryCompilerRequestProcessor.ImmediateProcessor immediate() { + public static QueryCompilerRequestProcessor.ImmediateProcessor immediate() { return new ImmediateProcessor(); } /** * @return A batch QueryCompilerRequestProcessor */ - static QueryCompilerRequestProcessor.BatchProcessor batch() { + public static QueryCompilerRequestProcessor.BatchProcessor batch() { return new BatchProcessor(); } @@ -45,16 +51,74 @@ static QueryCompilerRequestProcessor.BatchProcessor batch() { * @return a CachingSupplier that supplies a snapshot of the current query scope variables */ @VisibleForTesting - static CachingSupplier> newQueryScopeVariableSupplier() { + public static CachingSupplier> newQueryScopeVariableSupplier() { final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); return new CachingSupplier<>(() -> Collections.unmodifiableMap( queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)))); } + /** + * @return a CachingSupplier that supplies a snapshot of the current {@link QueryLibrary} package imports + */ + private static CachingSupplier> newPackageImportsSupplier() { + final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); + return new CachingSupplier<>(() -> Set.copyOf(queryLibrary.getPackageImports())); + } + + /** + * @return a CachingSupplier that supplies a snapshot of the current {@link QueryLibrary} class imports + */ + private static CachingSupplier>> newClassImportsSupplier() { + final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); + return new CachingSupplier<>(() -> { + final Collection> classImports = new HashSet<>(queryLibrary.getClassImports()); + // because QueryLibrary is in the context package, without visibility, we need to add these manually + classImports.add(TrackingWritableRowSet.class); + classImports.add(WritableColumnSource.class); + return Collections.unmodifiableCollection(classImports); + }); + } + + /** + * @return a CachingSupplier that supplies a snapshot of the current {@link QueryLibrary} static imports + */ + private static CachingSupplier>> newStaticImportsSupplier() { + final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); + return new CachingSupplier<>(() -> Set.copyOf(queryLibrary.getStaticImports())); + } + + private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); + private final CachingSupplier> packageImportsSupplier = newPackageImportsSupplier(); + private final CachingSupplier>> classImportsSupplier = newClassImportsSupplier(); + private final CachingSupplier>> staticImportsSupplier = newStaticImportsSupplier(); + /** * @return a lazily cached snapshot of the current query scope variables */ - Map getQueryScopeVariables(); + public final Map getQueryScopeVariables() { + return queryScopeVariableSupplier.get(); + } + + /** + * @return a lazily cached snapshot of the current {@link QueryLibrary} package imports + */ + public final Collection getPackageImports() { + return packageImportsSupplier.get(); + } + + /** + * @return a lazily cached snapshot of the current {@link QueryLibrary} class imports + */ + public final Collection> getClassImports() { + return classImportsSupplier.get(); + } + + /** + * @return a lazily cached snapshot of the current {@link QueryLibrary} static imports + */ + public final Collection> getStaticImports() { + return staticImportsSupplier.get(); + } /** * Submit a request for compilation. The QueryCompilerRequestProcessor is not required to immediately compile this @@ -62,24 +126,16 @@ static CachingSupplier> newQueryScopeVariableSupplier() { * * @param request the request to compile */ - CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); + public abstract CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); /** * A QueryCompilerRequestProcessor that immediately compiles requests. */ - class ImmediateProcessor implements QueryCompilerRequestProcessor { - - private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); - + public static class ImmediateProcessor extends QueryCompilerRequestProcessor { private ImmediateProcessor() { // force use of static factory method } - @Override - public Map getQueryScopeVariables() { - return queryScopeVariableSupplier.get(); - } - @Override public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { final String desc = "Compile: " + request.description(); @@ -108,20 +164,14 @@ public CompletionStageFuture> submit(@NotNull final QueryCompilerReques *

* The compile method must be called to actually compile the requests. */ - class BatchProcessor implements QueryCompilerRequestProcessor { + public static class BatchProcessor extends QueryCompilerRequestProcessor { private final List requests = new ArrayList<>(); private final List>> resolvers = new ArrayList<>(); - private final CachingSupplier> queryScopeVariableSupplier = newQueryScopeVariableSupplier(); private BatchProcessor() { // force use of static factory method } - @Override - public Map getQueryScopeVariables() { - return queryScopeVariableSupplier.get(); - } - @Override public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { final CompletionStageFuture.Resolver> resolver = CompletionStageFuture.make(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index dec6af575e2..4bb5e19dad4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -71,7 +71,6 @@ import io.deephaven.util.type.ArrayTypeUtils; import io.deephaven.vector.Vector; import org.apache.commons.lang3.mutable.Mutable; -import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -1498,7 +1497,7 @@ public Table update(final Collection newColumns) { */ public SelectValidationResult validateSelect(final SelectColumn... selectColumns) { final SelectColumn[] clones = SelectColumn.copyFrom(selectColumns); - SelectAndViewAnalyzer.AnalyzerContext analyzerContext = SelectAndViewAnalyzer.create( + SelectAndViewAnalyzer.AnalyzerContext analyzerContext = SelectAndViewAnalyzer.createContext( this, SelectAndViewAnalyzer.Mode.SELECT_STATIC, getModifiedColumnSetForUpdates(), true, false, clones); return new SelectValidationResult(analyzerContext.createAnalyzer(), clones); @@ -1526,7 +1525,7 @@ private Table selectOrUpdate(Flavor flavor, final SelectColumn... selectColumns) } } final boolean publishTheseSources = flavor == Flavor.Update; - final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = SelectAndViewAnalyzer.create( + final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = SelectAndViewAnalyzer.createContext( this, mode, getModifiedColumnSetForUpdates(), publishTheseSources, true, selectColumns); @@ -1557,23 +1556,10 @@ this, mode, getModifiedColumnSetForUpdates(), publishTheseSources, true, new SelectAndViewAnalyzer.UpdateHelper(emptyRowSet, fakeUpdate)) { try { - final MutableBoolean errorOccurred = new MutableBoolean(); - analyzer.applyUpdate( fakeUpdate, emptyRowSet, updateHelper, jobScheduler, liveResultCapture, - () -> { - if (errorOccurred.booleanValue()) { - return; - } - waitForResult.complete(null); - }, - err -> { - if (errorOccurred.booleanValue()) { - return; - } - errorOccurred.setTrue(); - waitForResult.completeExceptionally(err); - }); + () -> waitForResult.complete(null), + waitForResult::completeExceptionally); } catch (Exception e) { waitForResult.completeExceptionally(e); } @@ -1611,7 +1597,7 @@ this, mode, getModifiedColumnSetForUpdates(), publishTheseSources, true, resultTable.setFlat(); } propagateDataIndexes(processedColumns, resultTable); - for (final ColumnSource columnSource : analyzerContext.getSelectedColumnSources() + for (final ColumnSource columnSource : analyzerContext.getNewColumnSources() .values()) { if (columnSource instanceof PossiblyImmutableColumnSource) { ((PossiblyImmutableColumnSource) columnSource).setImmutable(); @@ -1778,7 +1764,7 @@ updateDescription, sizeForInstrumentation(), () -> { initializeWithSnapshot(humanReadablePrefix, sc, (usePrev, beforeClockValue) -> { final boolean publishTheseSources = flavor == Flavor.UpdateView; final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = - SelectAndViewAnalyzer.create( + SelectAndViewAnalyzer.createContext( this, SelectAndViewAnalyzer.Mode.VIEW_EAGER, getModifiedColumnSetForUpdates(), publishTheseSources, true, viewColumns); @@ -1869,9 +1855,10 @@ public Table lazyUpdate(final Collection newColumns) { sizeForInstrumentation(), () -> { checkInitiateOperation(); - final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = SelectAndViewAnalyzer.create( - this, SelectAndViewAnalyzer.Mode.VIEW_LAZY, - getModifiedColumnSetForUpdates(), true, true, selectColumns); + final SelectAndViewAnalyzer.AnalyzerContext analyzerContext = + SelectAndViewAnalyzer.createContext( + this, SelectAndViewAnalyzer.Mode.VIEW_LAZY, + getModifiedColumnSetForUpdates(), true, true, selectColumns); final SelectColumn[] processedColumns = analyzerContext.getProcessedColumns() .toArray(SelectColumn[]::new); final QueryTable result = new QueryTable( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java index 5e86ef5af5b..b84f5a0230f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/lang/QueryLanguageParser.java @@ -182,6 +182,8 @@ public final class QueryLanguageParser extends GenericVisitorAdapter, Q * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the * {@link #getResult()}} method. + *

+ * Note that the provided Collections and Maps must not be mutated concurrently with or after construction. * * @param expression The query language expression to parse * @param packageImports Wildcard package imports @@ -190,9 +192,10 @@ public final class QueryLanguageParser extends GenericVisitorAdapter, Q * imported. * @param variables A map of the names of scope variables to their types * @param variableTypeArguments A map of the names of scope variables to their type arguments - * @param unboxArguments If true it will unbox the query scope arguments * @param queryScopeVariables A mutable map of the names of query scope variables to their values * @param columnVariables A set of column variable names + * @param unboxArguments If true it will unbox the query scope arguments + * @param timeConversionResult The result of converting time literals in the expression * @throws QueryLanguageParseException If any exception or error is encountered */ public QueryLanguageParser( @@ -225,6 +228,8 @@ public QueryLanguageParser( * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the * {@link #getResult()}} method. + *

+ * Note that the provided Collections and Maps must not be mutated concurrently with or after construction. * * @param expression The query language expression to parse * @param packageImports Wildcard package imports @@ -247,6 +252,28 @@ public QueryLanguageParser( variableTypeArguments, null, null, true, null); } + /** + * Create a QueryLanguageParser and parse the given {@code expression}. After construction, the + * {@link QueryLanguageParser.Result result} of parsing the {@code expression} is available with the + * {@link #getResult()}} method. + *

+ * Note that the provided Collections and Maps must not be mutated concurrently with or after construction. + * + * @param expression The query language expression to parse + * @param packageImports Wildcard package imports + * @param classImports Individual class imports + * @param staticImports Wildcard static imports. All static variables and methods for the given classes are + * imported. + * @param variables A map of the names of scope variables to their types + * @param variableTypeArguments A map of the names of scope variables to their type arguments + * @param queryScopeVariables A mutable map of the names of query scope variables to their values + * @param columnVariables A set of column variable names + * @param unboxArguments If true it will unbox the query scope arguments + * @param verifyIdempotence If true, the parser will verify that the result expression will not mutate when parsed + * @param pyCallableWrapperImplName The name of the PyCallableWrapper implementation to use + * @param timeConversionResult The result of converting time literals in the expression + * @throws QueryLanguageParseException If any exception or error is encountered + */ @VisibleForTesting QueryLanguageParser( String expression, diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index e96bb529f1e..6c7de789aef 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -89,7 +89,11 @@ public synchronized void init( try { final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( formula, tableDefinition.getColumnNameMap(), outerToInnerNames, - compilationProcessor.getQueryScopeVariables(), unboxArguments); + compilationProcessor.getQueryScopeVariables(), + compilationProcessor.getPackageImports(), + compilationProcessor.getClassImports(), + compilationProcessor.getStaticImports(), + unboxArguments); formulaShiftColPair = result.getFormulaShiftColPair(); if (formulaShiftColPair != null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index 37224bd7628..ace064b9353 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -98,8 +98,12 @@ public List initInputs( return usedColumns; } - private HashMap> filterColumnSources( + private Map> filterColumnSources( final Map> columnsOfInterest) { + if (usedColumns.isEmpty() && usedColumnArrays.isEmpty()) { + return Map.of(); + } + final HashMap> sources = new HashMap<>(); for (String columnName : usedColumns) { sources.put(columnName, columnsOfInterest.get(columnName)); @@ -131,7 +135,7 @@ public void validateSafeForRefresh(BaseTable sourceTable) { } protected void applyUsedVariables( - @NotNull final Map> columnDefinitionMap, + @NotNull final Map> parentColumnDefinitions, @NotNull final Set variablesUsed, @NotNull final Map possibleParams) { // the column definition map passed in is being mutated by the caller, so we need to make a copy @@ -141,7 +145,7 @@ protected void applyUsedVariables( usedColumns = new ArrayList<>(); usedColumnArrays = new ArrayList<>(); for (String variable : variablesUsed) { - ColumnDefinition columnDefinition = columnDefinitionMap.get(variable); + ColumnDefinition columnDefinition = parentColumnDefinitions.get(variable); if (variable.equals("i")) { usesI = true; } else if (variable.equals("ii")) { @@ -154,7 +158,7 @@ protected void applyUsedVariables( } else { String strippedColumnName = variable.substring(0, Math.max(0, variable.length() - COLUMN_SUFFIX.length())); - columnDefinition = columnDefinitionMap.get(strippedColumnName); + columnDefinition = parentColumnDefinitions.get(strippedColumnName); if (variable.endsWith(COLUMN_SUFFIX) && columnDefinition != null) { columnDefinitions.put(strippedColumnName, columnDefinition); usedColumnArrays.add(strippedColumnName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index dd1e124e6de..c7e1c32ffdd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -181,7 +181,10 @@ public List initDef( try { final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( formulaString, columnDefinitionMap, Collections.emptyMap(), - compilationRequestProcessor.getQueryScopeVariables()); + compilationRequestProcessor.getQueryScopeVariables(), + compilationRequestProcessor.getPackageImports(), + compilationRequestProcessor.getClassImports(), + compilationRequestProcessor.getStaticImports()); analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, result); hasConstantValue = result.isConstantValueExpression(); formulaShiftColPair = result.getFormulaShiftColPair(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index 570b4412916..6e34e9b39e0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -76,22 +76,23 @@ public static void initializeSelectColumns( } } - public static AnalyzerContext create( - final QueryTable sourceTable, + public static AnalyzerContext createContext( + final QueryTable parentTable, final Mode mode, final ModifiedColumnSet parentMcs, - final boolean publishTheseSources, + final boolean publishParentSources, boolean useShiftedColumns, final SelectColumn... selectColumns) { - final UpdateGraph updateGraph = sourceTable.getUpdateGraph(); + final UpdateGraph updateGraph = parentTable.getUpdateGraph(); - final Map> columnSources = sourceTable.getColumnSourceMap(); - final TrackingRowSet rowSet = sourceTable.getRowSet(); + final Map> columnSources = parentTable.getColumnSourceMap(); + final TrackingRowSet rowSet = parentTable.getRowSet(); - final boolean flatResult = !sourceTable.isFlat() - && (columnSources.isEmpty() || !publishTheseSources) + final boolean parentIsFlat = parentTable.isFlat(); + final boolean flatResult = !parentIsFlat + && (columnSources.isEmpty() || !publishParentSources) && mode == Mode.SELECT_STATIC; - final AnalyzerContext context = new AnalyzerContext(sourceTable, publishTheseSources, flatResult); + final AnalyzerContext context = new AnalyzerContext(parentTable, publishParentSources, flatResult); final Map> columnDefinitions = new LinkedHashMap<>(); final RowRedirection rowRedirection; @@ -141,7 +142,7 @@ public static AnalyzerContext create( if (realColumn != null && !resultColumnNames.contains(realColumn.getSourceName())) { // if we are preserving a column, then we cannot change key space - context.flatResult &= !shouldPreserve(sc, columnSources.get(realColumn.getSourceName())); + context.flatResult &= !shouldPreserve(columnSources.get(realColumn.getSourceName())); } // TODO (deephaven#5760): If layers may define more than one column, we'll need to add all of them here. @@ -157,9 +158,10 @@ public static AnalyzerContext create( for (final SelectColumn sc : context.processedCols) { // if this select column depends on result column then its updates must happen in result-key-space - final boolean useResultKeySpace = !sourceTable.isFlat() && context.flatResult + // note: if flatResult is true then we are not preserving any parent columns + final boolean useResultKeySpace = context.flatResult && Stream.concat(sc.getColumns().stream(), sc.getColumnArrays().stream()) - .anyMatch(context.selectedSources::containsKey); + .anyMatch(context.newSources::containsKey); sc.initInputs(rowSet, useResultKeySpace ? context.allSourcesInResultKeySpace : context.allSources); @@ -179,8 +181,8 @@ public static AnalyzerContext create( // shifted columns appear to not be safe for refresh, so we do not validate them until they are rewritten // using the intermediary shifted column - if (sourceTable.isRefreshing()) { - sc.validateSafeForRefresh(sourceTable); + if (parentTable.isRefreshing()) { + sc.validateSafeForRefresh(parentTable); } if (hasConstantValue(sc)) { @@ -191,23 +193,20 @@ public static AnalyzerContext create( } final SourceColumn realColumn = tryToGetSourceColumn(sc); - if (realColumn != null && shouldPreserve(sc, sc.getDataView())) { - context.addLayer(new PreserveColumnLayer( - context, sc, sc.getDataView(), distinctDeps, mcsBuilder)); - continue; - } - - // look for an existing alias that can be preserved instead if (realColumn != null) { + if (shouldPreserve(sc.getDataView())) { + context.addLayer(new PreserveColumnLayer(context, sc, sc.getDataView(), distinctDeps, mcsBuilder)); + continue; + } + // look for an existing alias that can be preserved instead final ColumnSource alias = resultAlias.get(realColumn.getSourceName()); if (alias != null) { - context.addLayer(new PreserveColumnLayer( - context, sc, alias, distinctDeps, mcsBuilder)); + context.addLayer(new PreserveColumnLayer(context, sc, alias, distinctDeps, mcsBuilder)); continue; } } - // if this is a source column, then results are eligible for aliasing + // if this is a SourceColumn, then results are eligible for aliasing final Consumer> maybeCreateAlias = realColumn == null ? NOOP : cs -> resultAlias.put(realColumn.getSourceName(), cs); @@ -229,7 +228,7 @@ public static AnalyzerContext create( case SELECT_STATIC: { // We need to call newDestInstance because only newDestInstance has the knowledge to endow our // created array with the proper componentType (in the case of Vectors). - final WritableColumnSource scs = sourceTable.isFlat() || context.flatResult + final WritableColumnSource scs = parentIsFlat || context.flatResult ? sc.newFlatDestInstance(targetDestinationCapacity) : sc.newDestInstance(targetDestinationCapacity); maybeCreateAlias.accept(scs); @@ -326,11 +325,9 @@ private static boolean hasConstantValue(final SelectColumn sc) { return false; } - private static boolean shouldPreserve( - final SelectColumn sc, - final ColumnSource columnSource) { + private static boolean shouldPreserve(final ColumnSource columnSource) { return columnSource instanceof InMemoryColumnSource && ((InMemoryColumnSource) columnSource).isInMemory() - && !Vector.class.isAssignableFrom(sc.getReturnedType()); + && !Vector.class.isAssignableFrom(columnSource.getType()); } /** The layers that make up this analyzer. */ @@ -365,7 +362,7 @@ public final static class AnalyzerContext { /** The sources that are available to the analyzer, including parent columns, in result key space. */ private final Map> allSourcesInResultKeySpace; /** The sources that are explicitly defined to the analyzer, including preserved parent columns. */ - private final Map> selectedSources = new HashMap<>(); + private final Map> newSources = new HashMap<>(); /** The sources that are published to the child table. */ private final Map> publishedSources = new LinkedHashMap<>(); /** A mapping from result column name to the layer index that created it. */ @@ -385,10 +382,10 @@ public final static class AnalyzerContext { private int redirectionLayer = -1; AnalyzerContext( - final QueryTable sourceTable, + final QueryTable parentTable, final boolean publishTheseSources, final boolean flatResult) { - final Map> sources = sourceTable.getColumnSourceMap(); + final Map> sources = parentTable.getColumnSourceMap(); this.flatResult = flatResult; @@ -407,7 +404,7 @@ public final static class AnalyzerContext { } else { allSourcesInResultKeySpace = new HashMap<>(); - final RowRedirection rowRedirection = new WrappedRowSetRowRedirection(sourceTable.getRowSet()); + final RowRedirection rowRedirection = new WrappedRowSetRowRedirection(parentTable.getRowSet()); allSources.forEach((name, cs) -> allSourcesInResultKeySpace.put(name, RedirectedColumnSource.maybeRedirect(rowRedirection, cs))); } @@ -430,7 +427,7 @@ void addLayer(final Layer layer) { if (flatResult) { layer.populateColumnSources(allSourcesInResultKeySpace); } - layer.populateColumnSources(selectedSources); + layer.populateColumnSources(newSources); layer.populateColumnSources(publishedSources); layers.add(layer); @@ -478,7 +475,7 @@ void populateModifiedColumnSet( } else if (!allSources.containsKey(dep)) { // we should have blown up during initDef if this is the case throw new IllegalStateException("Column " + dep + " not found in any layer of the analyzer"); - } else if (!selectedSources.containsKey(dep)) { + } else if (!newSources.containsKey(dep)) { // this is a preserved parent column mcsBuilder.setAll(dep); } @@ -520,12 +517,12 @@ void setRedirectionLayer(final BitSet layerDependencies) { /** * @return the column sources explicitly created by the analyzer */ - public Map> getSelectedColumnSources() { - return selectedSources; + public Map> getNewColumnSources() { + return newSources; } /** - * @return the column sources that are published to the child table + * @return the column sources that are published through the child table */ public Map> getPublishedColumnSources() { // Note that if we have a shift column that we forcefully publish all columns. @@ -598,13 +595,13 @@ public Map calcEffects() { * Shift columns introduce intermediary table operations. This method applies remaining work to the result built * so far. * - * @param sourceTable the source table + * @param parentTable the source table * @param resultSoFar the intermediate result * @param updateFlavor the update flavor * @return the final result */ public QueryTable applyShiftsAndRemainingColumns( - final @NotNull QueryTable sourceTable, + final @NotNull QueryTable parentTable, @NotNull QueryTable resultSoFar, final UpdateFlavor updateFlavor) { if (shiftColumn != null) { @@ -613,19 +610,19 @@ public QueryTable applyShiftsAndRemainingColumns( } // shift columns may introduce modifies that are not present in the original table; set these before using - if (sourceTable.isRefreshing()) { - if (shiftColumn == null && sourceTable.isAddOnly()) { + if (parentTable.isRefreshing()) { + if (shiftColumn == null && parentTable.isAddOnly()) { resultSoFar.setAttribute(Table.ADD_ONLY_TABLE_ATTRIBUTE, true); } - if ((shiftColumn == null || !shiftColumnHasPositiveOffset) && sourceTable.isAppendOnly()) { + if ((shiftColumn == null || !shiftColumnHasPositiveOffset) && parentTable.isAppendOnly()) { // note if the shift offset is non-positive, then this result is still append-only resultSoFar.setAttribute(Table.APPEND_ONLY_TABLE_ATTRIBUTE, true); } - if (sourceTable.hasAttribute(Table.TEST_SOURCE_TABLE_ATTRIBUTE)) { + if (parentTable.hasAttribute(Table.TEST_SOURCE_TABLE_ATTRIBUTE)) { // be convenient for test authors by propagating the test source table attribute resultSoFar.setAttribute(Table.TEST_SOURCE_TABLE_ATTRIBUTE, true); } - if (sourceTable.isBlink()) { + if (parentTable.isBlink()) { // blink tables, although possibly not useful, can have shift columns resultSoFar.setAttribute(Table.BLINK_TABLE_ATTRIBUTE, true); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java index fe162c1f305..b617bb0283f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java @@ -3,8 +3,6 @@ // package io.deephaven.engine.table.impl.select.codegen; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.select.QueryScopeParamTypeUtil; @@ -12,8 +10,6 @@ import io.deephaven.vector.ObjectVector; import io.deephaven.engine.table.impl.select.DhFormulaColumn; import io.deephaven.engine.table.impl.select.formula.FormulaSourceDescriptor; -import io.deephaven.engine.table.WritableColumnSource; -import io.deephaven.engine.rowset.TrackingWritableRowSet; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import org.jetbrains.annotations.NotNull; @@ -83,8 +79,12 @@ public static QueryLanguageParser.Result parseFormula( @NotNull final String formulaString, @NotNull final Map> availableColumns, @NotNull final Map columnRenames, - @NotNull final Map queryScopeVariables) throws Exception { - return parseFormula(formulaString, availableColumns, columnRenames, queryScopeVariables, true); + @NotNull final Map queryScopeVariables, + @NotNull final Collection packageImports, + @NotNull final Collection> classImports, + @NotNull final Collection> staticImports) throws Exception { + return parseFormula(formulaString, availableColumns, columnRenames, queryScopeVariables, packageImports, + classImports, staticImports, true); } /** @@ -94,6 +94,9 @@ public static QueryLanguageParser.Result parseFormula( * @param availableColumns The columns available for use in the formula * @param columnRenames Outer to inner column name mapping * @param queryScopeVariables The query scope variables + * @param packageImports The package imports + * @param classImports The class imports + * @param staticImports The static imports * @param unboxArguments If true it will unbox the query scope arguments * @return The parsed formula {@link QueryLanguageParser.Result result} * @throws Exception If the formula cannot be parsed @@ -103,6 +106,9 @@ public static QueryLanguageParser.Result parseFormula( @NotNull final Map> availableColumns, @NotNull final Map columnRenames, @NotNull final Map queryScopeVariables, + @NotNull final Collection packageImports, + @NotNull final Collection> classImports, + @NotNull final Collection> staticImports, final boolean unboxArguments) throws Exception { final TimeLiteralReplacedExpression timeConversionResult = @@ -200,12 +206,8 @@ public static QueryLanguageParser.Result parseFormula( possibleVariables.putAll(timeConversionResult.getNewVariables()); - final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); - final Set> classImports = new HashSet<>(queryLibrary.getClassImports()); - classImports.add(TrackingWritableRowSet.class); - classImports.add(WritableColumnSource.class); - return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), queryLibrary.getPackageImports(), - classImports, queryLibrary.getStaticImports(), possibleVariables, possibleVariableParameterizedTypes, + return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), packageImports, + classImports, staticImports, possibleVariables, possibleVariableParameterizedTypes, queryScopeVariables, columnVariables, unboxArguments, timeConversionResult).getResult(); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java index c6b901a7dd3..0a02ddb31f9 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/array/BoxedBooleanArrayExpansionKernel.java @@ -47,7 +47,7 @@ public WritableChunk expand(final ObjectChunk source continue; } for (int j = 0; j < row.length; ++j) { - final byte value = row[j] ? BooleanUtils.TRUE_BOOLEAN_AS_BYTE : BooleanUtils.FALSE_BOOLEAN_AS_BYTE; + final byte value = BooleanUtils.booleanAsByte(row[j]); result.set(lenWritten + j, value); } lenWritten += row.length;