diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java new file mode 100644 index 00000000000..c410765e187 --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java @@ -0,0 +1,201 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * A safe version of CompletableFuture that does not expose the completion API. + * + * @param The result type returned by this future's {@code join} + */ +public interface CompletionStageFuture extends Future, CompletionStage { + + /** + * Returns a new CompletionStageFuture that is already completed with the given value. + * + * @param value the value + * @param the type of the value + * @return the completed CompletionStageFuture + * @see CompletableFuture#completedFuture(Object) + */ + static CompletionStageFuture completedFuture(U value) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + resolver.complete(value); + return resolver.getFuture(); + } + + /** + * Returns a new CompletableFuture that is already completed exceptionally with the given exception. + * + * @param ex the exception + * @param the type of the value + * @return the exceptionally completed CompletableFuture + * @since 9 + * @see CompletableFuture#failedFuture(Throwable) + */ + static CompletionStageFuture failedFuture(Throwable ex) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + resolver.completeExceptionally(ex); + return resolver.getFuture(); + } + + interface Resolver { + + /** + * If not already completed, sets the value returned by {@link #get()} and related methods to the given value. + * + * @param value the result value + * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#complete(Object) + */ + boolean complete(T value); + + /** + * If not already completed, causes invocations of {@link #get()} and related methods to throw the given + * exception. + * + * @param ex the exception + * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#completeExceptionally(Throwable) + */ + boolean completeExceptionally(@NotNull Throwable ex); + + /** + * @return the underlying future to provide to the recipient + */ + CompletionStageFuture getFuture(); + } + + @Override + CompletionStageFuture thenApply(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn, Executor executor); + + @Override + CompletionStageFuture thenAccept(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action, Executor executor); + + @Override + CompletionStageFuture thenRun(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCombine( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn, Executor executor); + + @Override + CompletionStageFuture thenAcceptBoth( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action, Executor executor); + + @Override + CompletionStageFuture runAfterBoth(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture applyToEither(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync( + CompletionStage other, Function fn, Executor executor); + + @Override + CompletionStageFuture acceptEither(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync( + CompletionStage other, Consumer action, Executor executor); + + @Override + CompletionStageFuture runAfterEither(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCompose(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync( + Function> fn, Executor executor); + + @Override + CompletionStageFuture handle(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn, Executor executor); + + @Override + CompletionStageFuture whenComplete(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action, Executor executor); + + @Override + CompletionStageFuture exceptionally(Function fn); +} diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java new file mode 100644 index 00000000000..d0ebef551fe --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java @@ -0,0 +1,356 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.function.*; + +/** + * A {@link CompletableFuture} that prevents users from completing the future by hiding behind + * {@link CompletionStageFuture}. + * + * @param The result type returned by this future's {@code join} + */ +@SuppressWarnings("unchecked") +public class CompletionStageFutureImpl extends CompletableFuture implements CompletionStageFuture { + + /** + * Create a new incomplete future. + * + * @param The result type returned by this future's {@code join} + * @return a resolver for the future + */ + public static Resolver make() { + return new CompletionStageFutureImpl().new ResolverImpl(); + } + + /** + * A resolver for this future implementation. + */ + private class ResolverImpl implements CompletionStageFuture.Resolver { + public boolean complete(final T value) { + return safelyComplete(value); + } + + public boolean completeExceptionally(@NotNull final Throwable ex) { + return safelyCompleteExceptionally(ex); + } + + public CompletionStageFuture getFuture() { + return CompletionStageFutureImpl.this; + } + } + + private boolean safelyComplete(final T value) { + return super.complete(value); + } + + private boolean safelyCompleteExceptionally(@NotNull final Throwable ex) { + return super.completeExceptionally(ex); + } + + @Override + public CompletableFuture newIncompleteFuture() { + return new CompletionStageFutureImpl<>(); + } + + /////////////////////////// + // CompletableFuture API // + /////////////////////////// + + @Override + public boolean complete(final T value) { + throw erroneousCompletionException(); + } + + @Override + public boolean completeExceptionally(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeValue(final T value) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeException(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync( + @NotNull final Supplier supplier, + @NotNull final Executor executor) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync(@NotNull final Supplier supplier) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeOnTimeout(final T value, long timeout, TimeUnit unit) { + throw erroneousCompletionException(); + } + + private static UnsupportedOperationException erroneousCompletionException() { + return new UnsupportedOperationException("Users should not complete futures."); + } + + ///////////////////////// + // CompletionStage API // + ///////////////////////// + + @Override + public CompletionStageFutureImpl thenApply(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApply(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync( + @NotNull final Function fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl thenAccept(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAccept(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync( + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenRun(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRun(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRunAsync(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync( + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenRunAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenCombine( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombine(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn, executor); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBoth( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBoth(other, action); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action, executor); + } + + + @Override + public CompletionStageFutureImpl runAfterBoth( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBoth(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl applyToEither( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEither(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn, Executor executor) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn, executor); + } + + @Override + public CompletionStageFutureImpl acceptEither( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEither(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl runAfterEither( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEither(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl thenCompose( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenCompose(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl handle(@NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handle(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handleAsync(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.handleAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl whenComplete(@NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenComplete(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl exceptionally(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.exceptionally(fn); + } +} diff --git a/engine/context/build.gradle b/engine/context/build.gradle index 0f683dcb5b9..da73cfac799 100644 --- a/engine/context/build.gradle +++ b/engine/context/build.gradle @@ -24,6 +24,7 @@ dependencies { implementation 'com.github.f4b6a3:uuid-creator:5.2.0' Classpaths.inheritCommonsText(project, 'implementation') + Classpaths.inheritImmutables(project) testImplementation TestTools.projectDependency(project, 'Base') testImplementation project(':engine-test-utils') diff --git a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java index 98cba3f36ee..dfc14a1bb5a 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java @@ -1,14 +1,13 @@ -/* - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending */ package io.deephaven.engine.context; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.ExecutionContextRegistrationException; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.File; -import java.util.Map; public class PoisonedQueryCompiler extends QueryCompiler { @@ -31,8 +30,9 @@ public void setParentClassLoader(ClassLoader parentClassLoader) { } @Override - public Class compile(@NotNull String className, @NotNull String classBody, @NotNull String packageNameRoot, - @Nullable StringBuilder codeLog, @NotNull Map> parameterClasses) { - return fail(); + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + fail(); } } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 6b54b9bb9fe..e7348de835f 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -5,16 +5,19 @@ import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.FileUtils; -import io.deephaven.base.Pair; import io.deephaven.configuration.Configuration; import io.deephaven.configuration.DataDir; import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +import io.deephaven.engine.updategraph.OperationInitializer; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.util.ByteUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.tools.*; import java.io.*; @@ -31,9 +34,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; @@ -41,6 +42,7 @@ import java.util.stream.Stream; public class QueryCompiler { + private static final Logger log = LoggerFactory.getLogger(QueryCompiler.class); /** * We pick a number just shy of 65536, leaving a little elbow room for good luck. @@ -56,9 +58,9 @@ public class QueryCompiler { private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; - private static final long codegenTimeoutMs = + private static final long CODEGEN_TIMEOUT_MS = Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); - private static final long codegenLoopDelayMs = + private static final long CODEGEN_LOOP_DELAY_MS = Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); @@ -76,7 +78,7 @@ static QueryCompiler createForUnitTests() { return new QueryCompiler(queryCompilerDir.toFile()); } - private final Map>> knownClasses = new HashMap<>(); + private final Map>> knownClasses = new HashMap<>(); private final String[] dynamicPatterns = new String[] {DYNAMIC_GROOVY_CLASS_PREFIX, FORMULA_PREFIX}; @@ -125,7 +127,7 @@ private QueryCompiler( /** * Enables or disables compilation logging. * - * @param logEnabled Whether or not logging should be enabled + * @param logEnabled Whether logging should be enabled * @return The value of {@code logEnabled} before calling this method. */ public static boolean setLogEnabled(boolean logEnabled) { @@ -204,66 +206,126 @@ public File getFakeClassDestination() { } public void setParentClassLoader(final ClassLoader parentClassLoader) { + // noinspection NonAtomicOperationOnVolatileField ucl = new WritableURLClassLoader(ucl.getURLs(), parentClassLoader); } - public final Class compile(String className, String classBody, String packageNameRoot) { - return compile(className, classBody, packageNameRoot, null, Collections.emptyMap()); - } - - public final Class compile(String className, String classBody, String packageNameRoot, - Map> parameterClasses) { - return compile(className, classBody, packageNameRoot, null, parameterClasses); + /** + * Compile a class. + * + * @param request The compilation request + */ + public Class compile(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + compile(request, resolver); + try { + return resolver.getFuture().get(); + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Could not compile class", e); + } } - public final Class compile(String className, String classBody, String packageNameRoot, StringBuilder codeLog) { - return compile(className, classBody, packageNameRoot, codeLog, Collections.emptyMap()); + /** + * Compile a class. + * + * @param request The compilation request + */ + public void compile( + @NotNull final QueryCompilerRequest request, + @NotNull final CompletionStageFuture.Resolver> resolver) { + // noinspection unchecked + compile(new QueryCompilerRequest[] {request}, new CompletionStageFuture.Resolver[] {resolver}); } /** - * Compile a class. + * Compiles all requests. * - * @param className Class name - * @param classBody Class body, before update with "$CLASS_NAME$" replacement and package name prefixing - * @param packageNameRoot Package name prefix - * @param codeLog Optional "log" for final class code - * @param parameterClasses Generic parameters, empty if none required - * @return The compiled class + * @param requests The compilation requests */ - public Class compile(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses) { - CompletableFuture> future; - final boolean alreadyExists; + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + if (requests.length == 0) { + return; + } + if (requests.length != resolvers.length) { + throw new IllegalArgumentException("Requests and resolvers must be the same length"); + } + + // noinspection unchecked + final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; + + final List newRequests = new ArrayList<>(); + final List>> newResolvers = new ArrayList<>(); synchronized (this) { - future = knownClasses.get(classBody); - if (future != null) { - alreadyExists = true; - } else { - future = new CompletableFuture<>(); - knownClasses.put(classBody, future); - alreadyExists = false; + for (int ii = 0; ii < requests.length; ++ii) { + final QueryCompilerRequest request = requests[ii]; + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + CompletionStageFuture> future = + knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); + if (future == null) { + newRequests.add(request); + newResolvers.add(resolver); + future = resolver.getFuture(); + } + allFutures[ii] = future; } } - // Someone else has already made the future. I'll just wait for the answer. - if (alreadyExists) { + /* + * @formatter:off + * 3. try to resolve CFs without compiling; retain next hash to try + * 4. compile all remaining with a single compilation task + * 5. goto step 3 + * 6. probably need Consumer> to fit DhFormulaColumn pattern? (other select columns don't need this) + * @formatter:on + */ + + if (!newResolvers.isEmpty()) { + // It's my job to fulfill the future of these futures. try { - return future.get(); - } catch (InterruptedException | ExecutionException error) { - throw new UncheckedDeephavenException(error); + compileHelper(newRequests, newResolvers); + } catch (RuntimeException e) { + // This is not a good state to be in, but we need to make sure we don't leave the resolvers hanging + newResolvers.forEach(f -> f.completeExceptionally(e)); + throw e; } } - // It's my job to fulfill the future. - try { - return compileHelper(className, classBody, packageNameRoot, codeLog, parameterClasses, future); - } catch (RuntimeException e) { - future.completeExceptionally(e); - throw e; + Error firstError = null; + RuntimeException firstException = null; + for (int ii = 0; ii < requests.length; ++ii) { + try { + resolvers[ii].complete(allFutures[ii].get()); + } catch (InterruptedException | ExecutionException e) { + final RuntimeException err; + if (e instanceof InterruptedException) { + err = new UncheckedDeephavenException("Interrupted while waiting for codegen", e); + } else { + Throwable cause = e.getCause(); + if (cause instanceof Error) { + firstError = (Error) cause; + resolvers[ii].completeExceptionally(cause); + continue; + } else if (cause instanceof RuntimeException) { + err = (RuntimeException) cause; + } else { + err = new UncheckedDeephavenException("Error during codegen", e); + } + } + if (firstException == null) { + firstException = err; + } + resolvers[ii].completeExceptionally(err); + } + } + if (firstError != null) { + throw firstError; + } + if (firstException != null) { + throw firstException; } } @@ -416,89 +478,164 @@ private String getClassPath() { return sb.toString(); } - private Class compileHelper(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses, - @NotNull final CompletableFuture> resultFuture) { + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers) { final MessageDigest digest; try { digest = MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); } - final String basicHashText = - ByteUtils.byteArrToHex(digest.digest(classBody.getBytes(StandardCharsets.UTF_8))); - - for (int pi = 0; pi < MAX_CLASS_COLLISIONS; ++pi) { - final String packageNameSuffix = "c_" + basicHashText - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - final String packageName = (packageNameRoot.isEmpty() - ? packageNameSuffix - : packageNameRoot + (packageNameRoot.endsWith(".") ? "" : ".") + packageNameSuffix); - final String fqClassName = packageName + "." + className; - - // Ask the classloader to load an existing class with this name. This might: - // 1. Fail to find a class (returning null) - // 2. Find a class whose body has the formula we are looking for - // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(fqClassName, parameterClasses); - if (result == null) { - // Couldn't find one, so try to create it. This might: - // A. succeed - // B. Lose a race to another process on the same file system which is compiling the identical formula - // C. Lose a race to another process on the same file system compiling a different formula that - // happens to have the same hash (same packageName). - // However, regardless of A-C, there will be *some* class being found (i.e. tryLoadClassByFqName won't - // return null). - maybeCreateClass(className, classBody, packageName, fqClassName); - - // We could be running on a screwy filesystem that is slow (e.g. NFS). - // If we wrote a file and can't load it ... then give the filesystem some time. - result = tryLoadClassByFqName(fqClassName, parameterClasses); + + + final String[] basicHashText = new String[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( + requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); + } + + int numCompiled = 0; + final int[] next_pi = new int[requests.size()]; + final boolean[] compiled = new boolean[requests.size()]; + final String[] packageName = new String[requests.size()]; + final String[] fqClassName = new String[requests.size()]; + + while (numCompiled < requests.size()) { + for (int ii = 0; ii < requests.size(); ++ii) { + if (compiled[ii]) { + continue; + } + + while (true) { + final int pi = next_pi[ii]++; + + final String packageNameSuffix = "c_" + basicHashText[ii] + + (pi == 0 ? "" : ("p" + pi)) + + "v" + JAVA_CLASS_VERSION; + + final QueryCompilerRequest request = requests.get(ii); + if (pi >= MAX_CLASS_COLLISIONS) { + throw new IllegalStateException("Found too many collisions for package name root " + + request.packageNameRoot() + ", class name=" + request.className() + ", class body " + + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); + } + + packageName[ii] = request.getPackageName(packageNameSuffix); + fqClassName[ii] = packageName[ii] + "." + request.className(); + + // Ask the classloader to load an existing class with this name. This might: + // 1. Fail to find a class (returning null) + // 2. Find a class whose body has the formula we are looking for + // 3. Find a class whose body has a different formula (hash collision) + Class result = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); + if (result == null) { + break; // we'll try to compile it + } + + if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolvers.get(ii), + result)) { + compiled[ii] = true; + ++numCompiled; + break; + } + } + } + + if (numCompiled == requests.size()) { + return; + } + + // Couldn't resolve at least one of the requests, so try a round of compilation. + final CompilationRequestAttempt[] compilationRequestAttempts = + new CompilationRequestAttempt[requests.size() - numCompiled]; + for (int ii = 0, jj = 0; ii < requests.size(); ++ii) { + if (!compiled[ii]) { + final QueryCompilerRequest request = requests.get(ii); + compilationRequestAttempts[jj++] = new CompilationRequestAttempt( + request, + packageName[ii], + fqClassName[ii], + resolvers.get(ii)); + } + } + + maybeCreateClass(compilationRequestAttempts); + + // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it + // ... then give the filesystem some time. All requests should use the same deadline. + final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; + for (int ii = 0; ii < requests.size(); ++ii) { + if (compiled[ii]) { + continue; + } + + final QueryCompilerRequest request = requests.get(ii); + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + compiled[ii] = true; + ++numCompiled; + continue; + } + + // This request may have: + // A. succeeded + // B. Lost a race to another process on the same file system which is compiling the identical formula + // C. Lost a race to another process on the same file system compiling a different formula that collides + + Class clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); try { - final long deadline = System.currentTimeMillis() + codegenTimeoutMs - codegenLoopDelayMs; - while (result == null && System.currentTimeMillis() < deadline) { - Thread.sleep(codegenLoopDelayMs); - result = tryLoadClassByFqName(fqClassName, parameterClasses); + while (clazz == null && System.currentTimeMillis() < deadline) { + // noinspection BusyWait + Thread.sleep(CODEGEN_LOOP_DELAY_MS); + clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); } - } catch (InterruptedException ignored) { - // we got interrupted, just quit looping and ignore it. + } catch (final InterruptedException ie) { + throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); } - if (result == null) { + // However, regardless of A-C, there will be *some* class being found + if (clazz == null) { throw new IllegalStateException("Should have been able to load *some* class here"); } - } - final String identifyingFieldValue = loadIdentifyingField(result); - - // If the class we found was indeed the class we were looking for, then complete the future and return it. - if (classBody.equals(identifyingFieldValue)) { - if (codeLog != null) { - // If the caller wants a textual copy of the code we either made, or just found in the cache. - codeLog.append(makeFinalCode(className, classBody, packageName)); - } - resultFuture.complete(result); - synchronized (this) { - // Note we are doing something kind of subtle here. We are removing an entry whose key was matched - // by value equality and replacing it with a value-equal but reference-different string that is a - // static member of the class we just loaded. This should be easier on the garbage collector because - // we are replacing a calculated value with a classloaded value and so in effect we are - // "canonicalizing" the string. This is important because these long strings stay in knownClasses - // forever. - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, resultFuture); - } - return result; + if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolver, clazz)) { + compiled[ii] = true; + ++numCompiled; + } } - // Try the next hash name } - throw new IllegalStateException("Found too many collisions for package name root " + packageNameRoot - + ", class name=" + className - + ", class body hash=" + basicHashText + " - contact Deephaven support!"); + } + + private boolean completeIfResultMatchesQueryCompilerRequest( + final String packageName, + final QueryCompilerRequest request, + final CompletionStageFuture.Resolver> resolver, + final Class result) { + final String identifyingFieldValue = loadIdentifyingField(result); + if (!request.classBody().equals(identifyingFieldValue)) { + return false; + } + + // If the caller wants a textual copy of the code we either made, or just found in the cache. + request.codeLog() + .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); + + // If the class we found was indeed the class we were looking for, then complete the future and return it. + resolver.complete(result); + + synchronized (this) { + // Note we are doing something kind of subtle here. We are removing an entry whose key was matched + // by value equality and replacing it with a value-equal but reference-different string that is a + // static member of the class we just loaded. This should be easier on the garbage collector because + // we are replacing a calculated value with a classloaded value and so in effect we are + // "canonicalizing" the string. This is important because these long strings stay in knownClasses + // forever. + knownClasses.remove(identifyingFieldValue); + knownClasses.put(identifyingFieldValue, resolver.getFuture()); + } + + return true; } private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { @@ -589,11 +726,19 @@ private static int calcBytesConsumed(final char ch) { } private static class JavaSourceFromString extends SimpleJavaFileObject { + final String description; final String code; + final CompletionStageFuture.Resolver> resolver; - JavaSourceFromString(String name, String code) { + JavaSourceFromString( + final String description, + final String name, + final String code, + final CompletionStageFuture.Resolver> resolver) { super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.description = description; this.code = code; + this.resolver = resolver; } public CharSequence getCharContent(boolean ignoreEncodingErrors) { @@ -601,56 +746,59 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) { } } - private static class JavaSourceFromFile extends SimpleJavaFileObject { - private static final int JAVA_LENGTH = Kind.SOURCE.extension.length(); - final String code; - - private JavaSourceFromFile(File basePath, File file) { - super(URI.create("string:///" + createName(basePath, file).replace('.', '/') + Kind.SOURCE.extension), - Kind.SOURCE); - try { - this.code = FileUtils.readTextFile(file); - } catch (IOException e) { - throw new UncheckedIOException(e); + private static class CompilationRequestAttempt { + final String description; + final String fqClassName; + final String finalCode; + final String packageName; + final String[] splitPackageName; + final QueryCompilerRequest request; + final CompletionStageFuture.Resolver> resolver; + + private CompilationRequestAttempt( + @NotNull final QueryCompilerRequest request, + @NotNull final String packageName, + @NotNull final String fqClassName, + @NotNull final CompletionStageFuture.Resolver> resolver) { + this.description = request.description(); + this.fqClassName = fqClassName; + this.resolver = resolver; + this.packageName = packageName; + this.request = request; + + finalCode = makeFinalCode(request.className(), request.classBody(), packageName); + + if (logEnabled) { + log.info().append("Generating code ").append(finalCode).endl(); } - } - private static String createName(File basePath, File file) { - final String base = basePath.getAbsolutePath(); - final String fileName = file.getAbsolutePath(); - if (!fileName.startsWith(base)) { - throw new IllegalArgumentException(file + " is not in " + basePath); + splitPackageName = packageName.split("\\."); + if (splitPackageName.length == 0) { + final Exception err = new UncheckedDeephavenException(String.format( + "packageName %s expected to have at least one .", packageName)); + resolver.completeExceptionally(err); } - final String basename = fileName.substring(base.length()); - if (basename.endsWith(".java")) { - return basename.substring(0, basename.length() - JAVA_LENGTH); - } else { - return basename; - } - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; } - } - private void maybeCreateClass(String className, String code, String packageName, String fqClassName) { - final String finalCode = makeFinalCode(className, code, packageName); + public void ensureDirectories(@NotNull final String rootPath) { + if (splitPackageName.length == 0) { + // we've already failed + return; + } - if (logEnabled) { - log.info().append("Generating code ").append(finalCode).endl(); + final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); + final File rpf = rootPathWithPackage.toFile(); + QueryCompiler.ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); } - final File ctxClassDestination = getClassDestination(); - - final String[] splitPackageName = packageName.split("\\."); - if (splitPackageName.length == 0) { - throw new UncheckedDeephavenException(String.format( - "packageName %s expected to have at least one .", packageName)); + public JavaSourceFromString makeSource() { + return new JavaSourceFromString(description, fqClassName, finalCode, resolver); } - final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + } + private void maybeCreateClass( + @NotNull final CompilationRequestAttempt[] requests) { // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 @@ -663,125 +811,172 @@ private void maybeCreateClass(String className, String code, String packageName, final String rootPathAsString; final String tempDirAsString; try { - rootPathAsString = ctxClassDestination.getAbsolutePath(); - final Path rootPathWithPackage = Paths.get(rootPathAsString, truncatedSplitPackageName); - final File rpf = rootPathWithPackage.toFile(); - ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); + rootPathAsString = getClassDestination().getAbsolutePath(); final Path tempPath = Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); tempDirAsString = tempPath.toFile().getAbsolutePath(); - } catch (IOException ioe) { - throw new UncheckedIOException(ioe); - } - try { - maybeCreateClassHelper(fqClassName, finalCode, splitPackageName, rootPathAsString, tempDirAsString); - } finally { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here + for (final CompilationRequestAttempt request : requests) { + request.ensureDirectories(rootPathAsString); + } + } catch (IOException ioe) { + Exception err = new UncheckedIOException(ioe); + for (final CompilationRequestAttempt request : requests) { + request.resolver.completeExceptionally(err); } + return; } - } - private void maybeCreateClassHelper(String fqClassName, String finalCode, String[] splitPackageName, - String rootPathAsString, String tempDirAsString) { - final StringWriter compilerOutput = new StringWriter(); final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); if (compiler == null) { throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); } - final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); - final List compilerOptions = Arrays.asList("-d", tempDirAsString, "-cp", classPathAsString); + final JavaFileManager fileManager = new SynchronizedJavaFileManager( + compiler.getStandardFileManager(null, null, null)); - final JavaFileManager fileManager = compiler.getStandardFileManager(null, null, null); - - boolean result = false; - boolean exceptionThrown = false; + boolean exceptionCaught = false; try { - result = compiler.getTask(compilerOutput, - fileManager, - null, - compilerOptions, - null, - Collections.singletonList(new JavaSourceFromString(fqClassName, finalCode))) - .call(); - } catch (final Throwable err) { - exceptionThrown = true; - throw err; + final OperationInitializer operationInitializer = ExecutionContext.getContext().getOperationInitializer(); + int parallelismFactor = operationInitializer.parallelismFactor(); + + int requestsPerTask = Math.max(32, (requests.length + parallelismFactor - 1) / parallelismFactor); + log.info().append("Compiling with parallelismFactor = ").append(parallelismFactor) + .append(" requestsPerTask = ").append(requestsPerTask).endl(); + if (parallelismFactor == 1 || requestsPerTask >= requests.length) { + maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + 0, requests.length, false); + } else { + int numTasks = (requests.length + requestsPerTask - 1) / requestsPerTask; + final Future[] tasks = new Future[numTasks]; + for (int jobId = 0; jobId < numTasks; ++jobId) { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.length, (jobId + 1) * requestsPerTask); + tasks[jobId] = operationInitializer.submit(() -> { + maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + startInclusive, endExclusive, false); + }); + } + for (int jobId = 0; jobId < numTasks; ++jobId) { + try { + tasks[jobId].get(); + } catch (Exception err) { + throw new UncheckedDeephavenException("Exception waiting for compilation task", err); + } + } + } + } catch (final Throwable t) { + exceptionCaught = true; + throw t; } finally { + try { + FileUtils.deleteRecursively(new File(tempDirAsString)); + } catch (Exception e) { + // ignore errors here + } + try { fileManager.close(); - } catch (final IOException ioe) { - if (!exceptionThrown) { + } catch (IOException ioe) { + if (!exceptionCaught) { // noinspection ThrowFromFinallyBlock throw new UncheckedIOException("Could not close JavaFileManager", ioe); } } } - if (!result) { - throw new UncheckedDeephavenException("Error compiling class " + fqClassName + ":\n" + compilerOutput); - } - // The above has compiled into e.g. - // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various - // class files} - // We want to atomically move it to e.g. - // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - Path srcDir = Paths.get(tempDirAsString, splitPackageName); - Path destDir = Paths.get(rootPathAsString, splitPackageName); - try { - Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException ioe) { - // The move might have failed for a variety of bad reasons. However, if the reason was because - // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore - // it. - if (!Files.exists(destDir)) { - throw new UncheckedIOException("Move failed for some reason other than destination already existing", - ioe); - } - } } - /** - * Try to compile the set of files, returning a pair of success and compiler output. - * - * @param basePath the base path for the java classes - * @param javaFiles the java source files - * @return a Pair of success, and the compiler output - */ - private Pair tryCompile(File basePath, Collection javaFiles) throws IOException { - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } + private void maybeCreateClassHelper( + @NotNull final JavaCompiler compiler, + @NotNull final JavaFileManager fileManager, + @NotNull final CompilationRequestAttempt[] requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive, + final boolean isRetry) { + final StringWriter compilerOutput = new StringWriter(); - final File outputDirectory = Files.createTempDirectory("temporaryCompilationDirectory").toFile(); + final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); + final List compilerOptions = Arrays.asList( + "-d", tempDirAsString, + "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail + "--should-stop=ifError=GENERATE"); + + final MutableInt numFailures = new MutableInt(0); + compiler.getTask(compilerOutput, + fileManager, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; + } + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source + numFailures.increment(); + } + }, + compilerOptions, + null, + Arrays.stream(requests, startInclusive, endExclusive) + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); try { - final StringWriter compilerOutput = new StringWriter(); - final String javaClasspath = getJavaClassPath(); + fileManager.close(); + } catch (IOException ioe) { + // ignore + } - final Collection javaFileObjects = javaFiles.stream() - .map(f -> new JavaSourceFromFile(basePath, f)).collect(Collectors.toList()); + final List shouldRetry; + if (!isRetry && numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive) { + // if this is the first attempt, and we had some failures, but not all of them failed, then we should retry + shouldRetry = new ArrayList<>(); + } else { + shouldRetry = null; + } - final boolean result = compiler.getTask(compilerOutput, null, null, - Arrays.asList("-d", outputDirectory.getAbsolutePath(), "-cp", - getClassPath() + File.pathSeparator + javaClasspath), - null, javaFileObjects).call(); + // The above has compiled into e.g. + // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various + // class files} + // We want to atomically move it to e.g. + // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} + Arrays.stream(requests, startInclusive, endExclusive).forEach(request -> { + final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); + final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); + try { + Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ioe) { + if (shouldRetry != null && !Files.exists(srcDir) && !request.resolver.getFuture().isDone()) { + // This source actually succeeded in compiling, but was not written because some other source failed + // to compile. Let's recursively call ourselves to try again. + shouldRetry.add(request); + return; + } - return new Pair<>(result, compilerOutput.toString()); - } finally { - FileUtils.deleteRecursively(outputDirectory); + if (!Files.exists(destDir) && !request.resolver.getFuture().isDone()) { + // The move might have failed for a variety of bad reasons. However, if the reason was because + // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore + // it. + request.resolver.completeExceptionally(new UncheckedIOException( + "Move failed for some reason other than destination already existing", ioe)); + } + } + }); + + if (shouldRetry != null && !shouldRetry.isEmpty()) { + maybeCreateClassHelper(compiler, fileManager, shouldRetry.toArray(CompilationRequestAttempt[]::new), + rootPathAsString, tempDirAsString, 0, shouldRetry.size(), true); } } /** - * Retrieve the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables. - * - * @return + * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables */ private static String getJavaClassPath() { String javaClasspath; @@ -792,14 +987,17 @@ private static String getJavaClassPath() { if (teamCityWorkDir != null) { // We are running in TeamCity, get the classpath differently final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); - - for (File f : classDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + if (classDirs != null) { + for (File f : classDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } } - final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); - for (File f : testDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); + if (testDirs != null) { + for (File f : testDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } } final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); @@ -835,7 +1033,7 @@ private static String getJavaClassPath() { // use the default path separator final String filePaths = Stream.of(extendedClassPath.split("file:/")) .map(String::trim) - .filter(fileName -> fileName.length() > 0) + .filter(fileName -> !fileName.isEmpty()) .collect(Collectors.joining(File.pathSeparator)); // Remove the classpath jar in question, and expand it with the files from the manifest diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java new file mode 100644 index 00000000000..38be0b97fe4 --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.context; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Immutable; + +import java.util.Map; +import java.util.Optional; + +/** + * A request to compile a java class. + */ +@Immutable +@BuildableStyle +public abstract class QueryCompilerRequest { + public static Builder builder() { + return ImmutableQueryCompilerRequest.builder(); + } + + /** + * @return the description to add to the query performance recorder nugget for this request + */ + public abstract String description(); + + /** + * @return the class name to use for the generated class + */ + public abstract String className(); + + /** + * @return the class body, before update with "$CLASS_NAME$" replacement and package name prefixing + */ + public abstract String classBody(); + + /** + * @return the package name prefix + */ + public abstract String packageNameRoot(); + + /** Optional "log" for final class code. */ + public abstract Optional codeLog(); + + /** + * @return the generic parameters, empty if none required + */ + public abstract Map> parameterClasses(); + + String getPackageName(final String packageNameSuffix) { + final String root = packageNameRoot(); + return root.isEmpty() + ? packageNameSuffix + : root + (root.endsWith(".") ? "" : ".") + packageNameSuffix; + } + + public interface Builder { + Builder description(String description); + + Builder className(String className); + + Builder classBody(String classBody); + + Builder packageNameRoot(String packageNameRoot); + + Builder codeLog(StringBuilder codeLog); + + Builder putAllParameterClasses(Map> parameterClasses); + + QueryCompilerRequest build(); + } +} diff --git a/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java new file mode 100644 index 00000000000..80a4d0c1667 --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java @@ -0,0 +1,132 @@ +package io.deephaven.engine.context.util; + +import javax.tools.FileObject; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Iterator; +import java.util.ServiceLoader; +import java.util.Set; + +public class SynchronizedJavaFileManager implements JavaFileManager { + + private final JavaFileManager delegate; + + public SynchronizedJavaFileManager(JavaFileManager delegate) { + this.delegate = delegate; + } + + @Override + public synchronized ClassLoader getClassLoader(Location location) { + return delegate.getClassLoader(location); + } + + @Override + public synchronized Iterable list( + Location location, + String packageName, + Set kinds, + boolean recurse) throws IOException { + return delegate.list(location, packageName, kinds, recurse); + } + + @Override + public synchronized String inferBinaryName(Location location, JavaFileObject file) { + return delegate.inferBinaryName(location, file); + } + + @Override + public synchronized boolean isSameFile(FileObject a, FileObject b) { + return delegate.isSameFile(a, b); + } + + @Override + public synchronized boolean handleOption(String current, Iterator remaining) { + return delegate.handleOption(current, remaining); + } + + @Override + public synchronized boolean hasLocation(Location location) { + return delegate.hasLocation(location); + } + + @Override + public synchronized JavaFileObject getJavaFileForInput( + Location location, + String className, + JavaFileObject.Kind kind) throws IOException { + return delegate.getJavaFileForInput(location, className, kind); + } + + @Override + public synchronized JavaFileObject getJavaFileForOutput( + Location location, + String className, + JavaFileObject.Kind kind, + FileObject sibling) throws IOException { + return delegate.getJavaFileForOutput(location, className, kind, sibling); + } + + @Override + public synchronized FileObject getFileForInput( + Location location, + String packageName, + String relativeName) throws IOException { + return delegate.getFileForInput(location, packageName, relativeName); + } + + @Override + public synchronized FileObject getFileForOutput( + Location location, + String packageName, + String relativeName, + FileObject sibling) throws IOException { + return delegate.getFileForOutput(location, packageName, relativeName, sibling); + } + + @Override + public synchronized void flush() throws IOException { + delegate.flush(); + } + + @Override + public synchronized void close() throws IOException { + delegate.close(); + } + + @Override + public synchronized int isSupportedOption(String option) { + return delegate.isSupportedOption(option); + } + + @Override + public synchronized Location getLocationForModule(Location location, String moduleName) throws IOException { + return delegate.getLocationForModule(location, moduleName); + } + + @Override + public synchronized Location getLocationForModule(Location location, JavaFileObject fo) throws IOException { + return delegate.getLocationForModule(location, fo); + } + + @Override + public synchronized ServiceLoader getServiceLoader(Location location, Class service) throws IOException { + return delegate.getServiceLoader(location, service); + } + + @Override + public synchronized String inferModuleName(Location location) throws IOException { + return delegate.inferModuleName(location); + } + + @Override + public synchronized Iterable> listLocationsForModules(Location location) throws IOException { + return delegate.listLocationsForModules(location); + } + + @Override + public synchronized boolean contains(Location location, FileObject fo) throws IOException { + return delegate.contains(location, fo); + } +} diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index 02374a67bf0..65d08b6f7f1 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -3,9 +3,13 @@ */ package io.deephaven.engine.context; +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.time.DateTimeUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; import io.deephaven.util.SafeCloseable; import org.junit.After; import org.junit.Before; @@ -18,8 +22,8 @@ import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; public class TestQueryCompiler { private final static int NUM_THREADS = 500; @@ -165,7 +169,7 @@ private void sleepIgnoringInterruptions(final long waitMillis) { } } - private void compile(boolean printDetails, final String className) throws Exception { + private void compile(boolean printDetails, final String className) { final long startMillis; if (printDetails) { startMillis = System.currentTimeMillis(); @@ -173,8 +177,13 @@ private void compile(boolean printDetails, final String className) throws Except } else { startMillis = 0; } - ExecutionContext.getContext().getQueryCompiler() - .compile(className, CLASS_CODE, "io.deephaven.temp"); + ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className(className) + .classBody(CLASS_CODE) + .packageNameRoot("io.deephaven.temp") + .build()); if (printDetails) { final long endMillis = System.currentTimeMillis(); System.out.println(printMillis(endMillis) + ": Thread 0 ending compile: (" + (endMillis - startMillis) @@ -201,8 +210,14 @@ public void testSimpleCompile() throws Exception { "}"); StringBuilder codeLog = new StringBuilder(); - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); @@ -224,21 +239,84 @@ public void testCollidingCompile() throws Exception { Thread t = new Thread(() -> { StringBuilder codeLog = new StringBuilder(); try { - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, - Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); } catch (Exception e) { - throw new RuntimeException(e); + throw new UncheckedDeephavenException(e); } }); t.start(); threads.add(t); } - for (int i = 0; i < threads.size(); ++i) { - threads.get(i).join(); + for (final Thread thread : threads) { + thread.join(); + } + } + + @Test + public void testMultiCompileWithFailure() throws ExecutionException, InterruptedException { + final String goodProgram = String.join( + "\n", + "public class GoodTest {", + " public static void main (String [] args) {", + " }", + "}"); + final String badProgram = String.join( + "\n", + "public class BadTest {", + " public static void main (String [] args) {", + " }", + "}}"); + + QueryCompilerRequest[] requests = new QueryCompilerRequest[] { + QueryCompilerRequest.builder() + .description("Test Bad Compile") + .className("BadTest") + .classBody(badProgram) + .packageNameRoot("com.deephaven.test") + .build(), + QueryCompilerRequest.builder() + .description("Test Good Compile") + .className("GoodTest") + .classBody(goodProgram) + .packageNameRoot("com.deephaven.test") + .build(), + }; + + // noinspection unchecked + CompletionStageFuture.Resolver>[] resolvers = + (CompletionStageFuture.Resolver>[]) new CompletionStageFuture.Resolver[] { + CompletionStageFutureImpl.make(), + CompletionStageFutureImpl.make(), + }; + + Exception firstErr; + try { + ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers); + // noinspection DataFlowIssue + throw Assert.statementNeverExecuted(); + } catch (Exception err) { + firstErr = err; + } + + Assert.eqTrue(resolvers[0].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.eqTrue(resolvers[1].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.neqNull(resolvers[1].getFuture().get(), "resolvers[1].getFuture().get()"); + try { + resolvers[0].getFuture().get(); + // noinspection DataFlowIssue + throw Assert.statementNeverExecuted(); + } catch (ExecutionException err) { + Assert.eq(firstErr, "firstErr", err.getCause(), "err"); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java index dc526030bfd..a9ce4f43d5a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java @@ -3,8 +3,11 @@ */ package io.deephaven.engine.table.impl; +import io.deephaven.api.ColumnName; import io.deephaven.api.Selectable; import io.deephaven.api.filter.Filter; +import io.deephaven.api.updateby.UpdateByControl; +import io.deephaven.api.updateby.UpdateByOperation; import io.deephaven.base.reference.SimpleReference; import io.deephaven.base.verify.Assert; import io.deephaven.datastructures.util.CollectionUtil; @@ -28,6 +31,12 @@ */ public class DeferredViewTable extends RedefinableTable { + @Override + public Table updateBy(@NotNull UpdateByControl control, @NotNull Collection ops, + @NotNull Collection byColumns) { + return super.updateBy(control, ops, byColumns); + } + private final TableReference tableReference; protected final String[] deferredDropColumns; protected final SelectColumn[] deferredViewColumns; @@ -46,16 +55,19 @@ public DeferredViewTable(@NotNull final TableDefinition definition, this.deferredViewColumns = deferredViewColumns == null ? SelectColumn.ZERO_LENGTH_SELECT_COLUMN_ARRAY : deferredViewColumns; final TableDefinition parentDefinition = tableReference.getDefinition(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); SelectAndViewAnalyzer.initializeSelectColumns( - parentDefinition.getColumnNameMap(), this.deferredViewColumns); + parentDefinition.getColumnNameMap(), this.deferredViewColumns, compilationProcessor); this.deferredFilters = deferredFilters == null ? WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY : deferredFilters; for (final WhereFilter sf : this.deferredFilters) { - sf.init(parentDefinition); + sf.init(parentDefinition, compilationProcessor); if (sf instanceof LivenessReferent && sf.isRefreshing()) { manage((LivenessReferent) sf); setRefreshing(true); } } + compilationProcessor.compile(); // we really only expect one of these things to be set! final boolean haveDrop = this.deferredDropColumns.length > 0; @@ -78,9 +90,12 @@ public DeferredViewTable(@NotNull final TableDefinition definition, @Override public Table where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter f : whereFilters) { - f.init(definition); + f.init(definition, compilationProcessor); } + compilationProcessor.compile(); return getResultTableWithWhere(whereFilters); } @@ -189,8 +204,10 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { } } + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final WhereFilter filter : filters) { - filter.init(definition); + filter.init(definition, compilationProcessor); final boolean isPostView = Stream.of(filter.getColumns(), filter.getColumnArrays()) .flatMap(Collection::stream) @@ -220,6 +237,7 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { postViewFilters.add(filter); } } + compilationProcessor.compile(); return new PreAndPostFilters(preViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY), postViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY)); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java index 166e088657d..6a3e83ccad0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java @@ -280,8 +280,10 @@ private Table whereImpl(final WhereFilter[] whereFilters) { Set groupingColumnNames = groupingColumns.stream().map(ColumnDefinition::getName).collect(Collectors.toSet()); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, compilationProcessor); List columns = whereFilter.getColumns(); if (whereFilter instanceof ReindexingFilter) { otherFilters.add(whereFilter); @@ -294,6 +296,7 @@ private Table whereImpl(final WhereFilter[] whereFilters) { otherFilters.add(whereFilter); } } + compilationProcessor.compile(); // if there was nothing that actually required the partition, defer the result. if (partitionFilters.isEmpty()) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java new file mode 100644 index 00000000000..3c72a36332a --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl; + +import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +public interface QueryCompilerRequestProcessor { + /** + * Submit a request for compilation. The QueryCompilerRequestProcessor is not required to immediately compile this + * request. + * + * @param request the request to compile + */ + CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); + + /** + * A QueryCompilerRequestProcessor that immediately compiles requests. + */ + class ImmediateProcessor implements QueryCompilerRequestProcessor { + public static final ImmediateProcessor INSTANCE = new ImmediateProcessor(); + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final String desc = "Compile: " + request.description(); + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + ExecutionContext.getContext().getQueryCompiler().compile(request, resolver); + } + return resolver.getFuture(); + } + } + + /** + * A QueryCompilerRequestProcessor that batches requests and compiles them all at once. + *

+ * The compile method must be called to actually compile the requests. + */ + class BatchProcessor implements QueryCompilerRequestProcessor { + private final List requests = new ArrayList<>(); + private final List>> resolvers = new ArrayList<>(); + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + requests.add(request); + resolvers.add(resolver); + return resolver.getFuture(); + } + + /** + * Compile all the requests that have been submitted. + */ + public void compile() { + if (requests.isEmpty()) { + return; + } + + final String desc; + if (requests.size() == 1) { + desc = "Compile: " + requests.get(0).description(); + } else { + final StringBuilder descriptionBuilder = new StringBuilder(); + descriptionBuilder.append("Batch Compile of ").append(requests.size()).append(" requests:\n"); + for (final QueryCompilerRequest request : requests) { + descriptionBuilder.append('\t').append(request.description()).append('\n'); + } + desc = descriptionBuilder.toString(); + } + + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); + if (requests.size() == 1) { + compiler.compile(requests.get(0), resolvers.get(0)); + } else { + compiler.compile( + requests.toArray(QueryCompilerRequest[]::new), + resolvers.toArray(CompletionStageFuture.Resolver[]::new)); + } + } + } + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index 72c4d20aff0..8b6c01aa9dd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -1218,8 +1218,10 @@ private QueryTable whereInternal(final WhereFilter... filters) { List selectFilters = new LinkedList<>(); List>>> shiftColPairs = new LinkedList<>(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final WhereFilter filter : filters) { - filter.init(getDefinition()); + filter.init(getDefinition(), compilationProcessor); if (filter instanceof AbstractConditionFilter && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); @@ -1227,6 +1229,7 @@ private QueryTable whereInternal(final WhereFilter... filters) { selectFilters.add(filter); } } + compilationProcessor.compile(); if (!shiftColPairs.isEmpty()) { return (QueryTable) ShiftedColumnsFactory.where(this, shiftColPairs, selectFilters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java index ed787d1baab..ac572277b96 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java @@ -5,10 +5,13 @@ import io.deephaven.api.Selectable; import io.deephaven.api.Pair; +import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.*; +import io.deephaven.util.SafeCloseable; import org.jetbrains.annotations.NotNull; import java.util.*; @@ -49,8 +52,11 @@ private Table viewInternal(Collection selectables, boolean final Map> resultColumnsExternal = new LinkedHashMap<>(); final Map> allColumns = new HashMap<>(definition.getColumnNameMap()); boolean simpleRetain = true; + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final SelectColumn selectColumn : columns) { - List usedColumnNames = selectColumn.initDef(allColumns); + List usedColumnNames = selectColumn.initDef(allColumns, compilationProcessor); usedColumnNames.addAll(selectColumn.getColumnArrays()); resultColumnsInternal.addAll(usedColumnNames.stream() .filter(usedColumnName -> !resultColumnsExternal.containsKey(usedColumnName)) @@ -66,6 +72,9 @@ private Table viewInternal(Collection selectables, boolean allColumns.put(selectColumn.getName(), columnDef); } + // compile all formulas at once + compilationProcessor.compile(); + TableDefinition newDefExternal = TableDefinition.of( resultColumnsExternal.values().toArray(ColumnDefinition.ZERO_LENGTH_COLUMN_DEFINITION_ARRAY)); if (simpleRetain) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java index cb2b169d7d6..3c69f7d2a4a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java @@ -91,9 +91,11 @@ public Result initialize(boolean usePrev, long beforeClock) { final List dependencies = new ArrayList<>(); final Map> newColumns = new LinkedHashMap<>(parent.getColumnSourceMap()); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); matchColumns.forEach(holder -> { final WhereFilter filter = holder.getFilter(); - filter.init(parent.getDefinition()); + filter.init(parent.getDefinition(), compilationProcessor); final WritableRowSet result = filter.filter(fullRowSet, fullRowSet, parent, usePrev); holder.column = new IndexWrapperColumnSource( holder.getColumnName(), parent, result.toTracking(), filter); @@ -115,6 +117,7 @@ public Result initialize(boolean usePrev, long beforeClock) { anyRefreshing.setTrue(); } }); + compilationProcessor.compile(); this.resultTable = new QueryTable(parent.getRowSet(), newColumns); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java index 3aef55a0c50..dae929b74dc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java @@ -38,11 +38,8 @@ import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TupleSourceFactory; import io.deephaven.engine.table.impl.by.rollup.NullColumns; import io.deephaven.engine.table.impl.by.rollup.RollupAggregation; import io.deephaven.engine.table.impl.by.rollup.RollupAggregationOutputs; @@ -282,7 +279,13 @@ public AggregationContext makeAggregationContext( @NotNull final String... groupByColumnNames) { switch (type) { case NORMAL: - return new NormalConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final AggregationContext aggContext = + new NormalConverter(table, requireStateChangeRecorder, compilationProcessor, groupByColumnNames) + .build(); + compilationProcessor.compile(); + return aggContext; case ROLLUP_BASE: return new RollupBaseConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); case ROLLUP_REAGGREGATED: @@ -664,12 +667,15 @@ final void addWeightedAvgOrSumOperator( * {@link AggregationContext} for standard aggregations. Accumulates state by visiting each aggregation. */ private final class NormalConverter extends Converter { + private final QueryCompilerRequestProcessor compilationProcessor; private NormalConverter( @NotNull final Table table, final boolean requireStateChangeRecorder, + @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final String... groupByColumnNames) { super(table, requireStateChangeRecorder, groupByColumnNames); + this.compilationProcessor = compilationProcessor; } // ------------------------------------------------------------------------------------------------------------- @@ -744,7 +750,8 @@ public void visit(@NotNull final AggSpecFormula formula) { final GroupByChunkedOperator groupByChunkedOperator = new GroupByChunkedOperator(table, false, null, resultPairs.stream().map(pair -> MatchPair.of((Pair) pair.input())).toArray(MatchPair[]::new)); final FormulaChunkedOperator formulaChunkedOperator = new FormulaChunkedOperator(groupByChunkedOperator, - true, formula.formula(), formula.paramToken(), MatchPair.fromPairs(resultPairs)); + true, formula.formula(), formula.paramToken(), compilationProcessor, + MatchPair.fromPairs(resultPairs)); addNoInputOperator(formulaChunkedOperator); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java index 7f489a060b6..5e8d07ad1ea 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.ChunkSource.GetContext; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaUtil; import io.deephaven.engine.liveness.LivenessReferent; import io.deephaven.engine.table.ModifiedColumnSet; @@ -69,10 +70,12 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { * @param columnParamName The token to substitute column names for * @param resultColumnPairs The names for formula input and result columns */ - FormulaChunkedOperator(@NotNull final GroupByChunkedOperator groupBy, + FormulaChunkedOperator( + @NotNull final GroupByChunkedOperator groupBy, final boolean delegateToBy, @NotNull final String formula, @NotNull final String columnParamName, + @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final MatchPair... resultColumnPairs) { this.groupBy = groupBy; this.delegateToBy = delegateToBy; @@ -95,9 +98,10 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(inputColumnName, inputColumnSource.getType(), inputColumnSource.getComponentType()); - formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition)); - // noinspection unchecked - resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource(0, formulaColumn.getReturnedType()); + formulaColumn.initDef( + Collections.singletonMap(inputColumnName, inputColumnDefinition), compilationProcessor); + resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource( + 0, formulaColumn.getReturnedType(), formulaColumn.getReturnedComponentType()); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java index 601ae2fb3bc..178aca03f04 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java @@ -11,6 +11,7 @@ import io.deephaven.chunk.util.hashing.CharChunkHasher; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetBuilderRandom; @@ -19,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MultiJoinModifiedSlotTracker; import io.deephaven.engine.table.impl.NaturalJoinModifiedSlotTracker; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.asofjoin.RightIncrementalAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.StaticAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.TypedAsOfJoinFactory; @@ -566,12 +568,18 @@ public static T make(HasherConfig hasherConfig, ColumnSource[] tableKe final String javaString = Arrays.stream(javaStrings).filter(s -> !s.startsWith("package ")).collect(Collectors.joining("\n")); - final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(className, javaString, - "io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen"); + final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(QueryCompilerRequest.builder() + .description("TypedHasherFactory: " + className) + .className(className) + .classBody(javaString) + .packageNameRoot("io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen") + .build()); + if (!hasherConfig.baseClass.isAssignableFrom(clazz)) { throw new IllegalStateException("Generated class is not a " + hasherConfig.baseClass.getCanonicalName()); } + // noinspection unchecked final Class castedClass = (Class) clazz; T retVal; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java index 7e3da401317..958915a15ab 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java @@ -9,10 +9,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.hierarchical.TreeTable; -import io.deephaven.engine.table.impl.AbsoluteSortColumnConventions; -import io.deephaven.engine.table.impl.NoSuchColumnException; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TableAdapter; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; @@ -281,16 +278,24 @@ private Stream absoluteSelectColumns() { // custom columns in the future. For now, we've plumbed absolute column value sorting via naming // conventions. Note that we simply avoid telling the client about these columns when sending schemas, so we // have no need to drop them post-sort. - return sortColumns.stream() + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + + final SelectColumn[] columns = sortColumns.stream() .map(sc -> sc.column().name()) .filter(AbsoluteSortColumnConventions::isAbsoluteColumnName) .map(cn -> { final String baseColumnName = AbsoluteSortColumnConventions.absoluteColumnNameToBaseName(cn); final Selectable selectable = AbsoluteSortColumnConventions.makeSelectable(cn, baseColumnName); final SelectColumn selectColumn = SelectColumn.of(selectable); - selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName))); + selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName)), + compilationProcessor); return selectColumn; - }); + }).toArray(SelectColumn[]::new); + + compilationProcessor.compile(); + return Stream.of(columns); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java index 0dc6f9b7be0..e92d6cad520 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.impl.BaseTable.CopyAttributeOperation; import io.deephaven.engine.table.impl.NotificationStepSource; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.SortOperation; import io.deephaven.engine.table.impl.by.AggregationProcessor; @@ -272,8 +273,10 @@ public static WhereFilter[] initializeAndValidateFilters( @NotNull final Collection filters, @NotNull final Function exceptionFactory) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final WhereFilter whereFilter : whereFilters) { - whereFilter.init(source.getDefinition()); + whereFilter.init(source.getDefinition(), compilationProcessor); final List invalidColumnsUsed = whereFilter.getColumns().stream().map(ColumnName::of) .filter(cn -> !groupByColumns.contains(cn)).map(ColumnName::name).collect(Collectors.toList()); if (!invalidColumnsUsed.isEmpty()) { @@ -287,6 +290,8 @@ public static WhereFilter[] initializeAndValidateFilters( + " may not use column arrays, but uses column arrays from " + whereFilter.getColumnArrays()); } } + compilationProcessor.compile(); + return whereFilters; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java index 875a7322f68..72271634a43 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java @@ -6,12 +6,12 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.TreeTable; import io.deephaven.engine.table.hierarchical.TreeTable.NodeOperationsRecorder; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -134,7 +134,14 @@ public Table where(Filter filter) { } private Stream whereFilters() { - return Stream.of(WhereFilter.fromInternal(filter)).peek(wf -> wf.init(getDefinition())); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final WhereFilter[] filters = WhereFilter.fromInternal(filter); + for (final WhereFilter filter : filters) { + filter.init(getDefinition(), compilationProcessor); + } + compilationProcessor.compile(); + return Stream.of(filters); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java index 9cee5c0eed1..258dbd3624a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java @@ -151,7 +151,11 @@ private TreeTableFilter(@NotNull final TreeTableImpl tree, @NotNull final WhereF parentIdColumnName = tree.getParentIdentifierColumn(); sourceRowLookup = tree.getSourceRowLookup(); this.filters = filters; - Arrays.stream(filters).forEach((final WhereFilter filter) -> filter.init(source.getDefinition())); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + Arrays.stream(filters) + .forEach((final WhereFilter filter) -> filter.init(source.getDefinition(), compilationProcessor)); + compilationProcessor.compile(); idSource = source.getColumnSource(tree.getIdentifierColumn().name()); parentIdSource = source.getColumnSource(tree.getParentIdentifierColumn().name()); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java index bf56bf038fc..7effc491fda 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java @@ -151,13 +151,17 @@ public TreeTable withFilter(@NotNull Filter filter) { if (whereFilters.length == 0) { return noopResult(); } + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); final Map> nodeSuitabilityToFilters = Stream.of(whereFilters) - .peek(wf -> wf.init(source.getDefinition())) + .peek(wf -> wf.init(source.getDefinition(), compilationProcessor)) .collect(Collectors.partitioningBy(wf -> { // Node-level filters have only node-filter columns and use no column arrays return wf.getColumns().stream().map(ColumnName::of).allMatch(nodeFilterColumns::contains) && wf.getColumnArrays().isEmpty(); })); + compilationProcessor.compile(); + final List nodeFilters = nodeSuitabilityToFilters.get(true); final List sourceFilters = nodeSuitabilityToFilters.get(false); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java index be54278bcd9..e7f30e70141 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java @@ -28,6 +28,12 @@ public final Class getReturnedType() { return Table.class; } + @Override + public Class getReturnedComponentType() { + // Table does not have a component type + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java index 796cef1b07b..fb4beb9812d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -57,7 +58,9 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); validateInputColumnDefinition(secondInputColumnName, columnDefinitionMap); return getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java index 14d476211bf..e9c93e37e37 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.LongSingleValueSource; @@ -44,7 +45,9 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { return getColumns(); } @@ -74,6 +77,12 @@ public final Class getReturnedType() { return long.class; } + @Override + public Class getReturnedComponentType() { + // long does not have a component type + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java index f6b39838ad4..f9abd5ebb05 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java @@ -19,10 +19,7 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.*; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.MatchPair; -import io.deephaven.engine.table.impl.MemoizedOperationKey; -import io.deephaven.engine.table.impl.QueryTable; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.engine.table.impl.select.MatchFilter; import io.deephaven.engine.table.impl.select.WhereFilter; @@ -235,10 +232,13 @@ private Map computeSharedAttributes(@NotNull final Iterator filters) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); final boolean invalidFilter = Arrays.stream(whereFilters).flatMap((final WhereFilter filter) -> { - filter.init(table.getDefinition()); + filter.init(table.getDefinition(), compilationProcessor); return Stream.concat(filter.getColumns().stream(), filter.getColumnArrays().stream()); }).anyMatch((final String columnName) -> columnName.equals(constituentColumnName)); + compilationProcessor.compile(); if (invalidFilter) { throw new IllegalArgumentException("Unsupported filter against constituent column " + constituentColumnName + " found in filters: " + filters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java index 5bfa7de3160..cad1a5696d5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java @@ -464,9 +464,12 @@ public PartitionedTable.Proxy sort(Collection columnsToSortBy) { public PartitionedTable.Proxy where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); final TableDefinition definition = target.constituentDefinition(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, compilationProcessor); } + compilationProcessor.compile(); return basicTransform(ct -> ct.where(Filter.and(WhereFilter.copyFrom(whereFilters)))); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java index 6a7cb5db1c1..d32cc90ce87 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java @@ -12,6 +12,7 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -52,7 +53,9 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); return getColumns(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java index cfadcba1fd2..26f8f20c1ab 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java @@ -14,6 +14,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.select.WhereFilterImpl; @@ -47,7 +48,9 @@ public List getColumnArrays() { } @Override - public void init(@NotNull final TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final ColumnDefinition columnDefinition = tableDefinition.getColumn(columnName.name()); if (columnDefinition == null) { throw new IllegalArgumentException(String.format("Missing expected input column %s", diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index fdbb01bedac..deae6bc994c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -11,25 +11,21 @@ import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.exceptions.SnapshotUnsuccessfulException; +import io.deephaven.engine.table.*; import io.deephaven.engine.updategraph.*; import io.deephaven.engine.rowset.*; -import io.deephaven.engine.table.SharedContext; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.util.*; import io.deephaven.engine.updategraph.NotificationQueue.Dependency; import io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph; import io.deephaven.io.log.LogEntry; -import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.exceptions.CancellationException; -import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.TableDefinition; import io.deephaven.util.datastructures.LongSizedDataStructure; import io.deephaven.engine.liveness.LivenessManager; import io.deephaven.engine.liveness.LivenessScope; import io.deephaven.engine.liveness.LivenessScopeStack; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.NotificationStepSource; -import io.deephaven.engine.table.ColumnSource; import io.deephaven.chunk.*; import io.deephaven.util.SafeCloseable; import io.deephaven.UncheckedDeephavenException; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index 66a8513fa4c..414768b0248 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -15,6 +15,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.select.python.ArgumentsChunked; import io.deephaven.engine.table.impl.select.python.DeephavenCompatibleFunction; @@ -80,7 +81,9 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } @@ -225,7 +228,7 @@ public synchronized void init(TableDefinition tableDefinition) { final Class resultType = result.getType(); checkReturnType(result, resultType); - generateFilterCode(tableDefinition, timeConversionResult, result); + generateFilterCode(tableDefinition, timeConversionResult, result, compilationProcessor); initialized = true; } } catch (Exception e) { @@ -302,9 +305,12 @@ private void checkReturnType(QueryLanguageParser.Result result, Class resultT } } - protected abstract void generateFilterCode(TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) throws MalformedURLException, ClassNotFoundException; + protected abstract void generateFilterCode( + @NotNull TableDefinition tableDefinition, + @NotNull TimeLiteralReplacedExpression timeConversionResult, + @NotNull QueryLanguageParser.Result result, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) + throws MalformedURLException, ClassNotFoundException; @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index 791be4beaf9..c0aecf5186f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -3,12 +3,15 @@ */ package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.verify.Require; import io.deephaven.configuration.Configuration; import io.deephaven.engine.table.*; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.vector.*; import io.deephaven.engine.table.impl.select.formula.*; @@ -21,6 +24,10 @@ import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; /** @@ -38,7 +45,7 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { @NotNull protected final String columnName; - protected FormulaFactory formulaFactory; + protected Future formulaFactory; private Formula formula; protected QueryScopeParam[] params; protected Map> columnSources; @@ -69,6 +76,11 @@ public Class getReturnedType() { return returnedType; } + @Override + public Class getReturnedComponentType() { + return returnedType.getComponentType(); + } + @Override public List initInputs( @NotNull final TrackingRowSet rowSet, @@ -79,6 +91,9 @@ public List initInputs( if (usedColumns != null) { return usedColumns; } + + // we'll have to assume that initDef has already been invoked if we could have grouped compilation requests + // otherwise this call will compile immediately if necessary return initDef(extractDefinitions(columnsOfInterest)); } @@ -220,7 +235,15 @@ private ColumnSource getViewColumnSource(boolean lazy) { private Formula getFormula(boolean initLazyMap, Map> columnsToData, QueryScopeParam... params) { - formula = formulaFactory.createFormula(rowSet, initLazyMap, columnsToData, params); + try { + // the future must already be completed or else it is an error + formula = formulaFactory.get(0, TimeUnit.SECONDS) + .createFormula(rowSet, initLazyMap, columnsToData, params); + } catch (TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Error creating formula for " + columnName, e); + } return formula; } @@ -254,10 +277,11 @@ private static Vector makeAppropriateVectorWrapper(ColumnSource cs, RowSet return new ObjectVectorColumnWrapper<>((ColumnSource) cs, rowSet); } - protected FormulaFactory createKernelFormulaFactory(final FormulaKernelFactory formulaKernelFactory) { + protected Future createKernelFormulaFactory( + @NotNull final CompletionStageFuture formulaKernelFactoryFuture) { final FormulaSourceDescriptor sd = getSourceDescriptor(); - return (rowSet, lazy, columnsToData, params) -> { + return formulaKernelFactoryFuture.thenApply(formulaKernelFactory -> (rowSet, lazy, columnsToData, params) -> { // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't mean // "cached", which is how we are using it. final Map> netColumnSources = new HashMap<>(); @@ -273,7 +297,7 @@ protected FormulaFactory createKernelFormulaFactory(final FormulaKernelFactory f } final FormulaKernel fk = formulaKernelFactory.createInstance(vectors, params); return new FormulaKernelAdapter(rowSet, sd, netColumnSources, fk); - }; + }); } protected abstract FormulaSourceDescriptor getSourceDescriptor(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java index 065558b0f8a..3de338e75ce 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.util.QueryConstants; @@ -67,7 +68,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { initialized = true; if (!started) { return; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java index 0527375f786..6663739cf70 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ByteRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -68,7 +69,9 @@ static WhereFilter makeByteRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java index 03a6dab0630..60ed8feb088 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.CharRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -63,7 +64,9 @@ static WhereFilter makeCharRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java index 400541e7069..c4488aabf1c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageFunctionUtils; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.updategraph.DynamicNode; @@ -46,7 +47,9 @@ public ClockFilter(@NotNull final String columnName, @NotNull final Clock clock, } @Override - public final void init(@NotNull final TableDefinition tableDefinition) {} + public final void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @Override public final List getColumns() { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java index 227d53094fb..18b76b6e27d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -41,7 +42,9 @@ public static ComparableRangeFilter makeForTest(String columnName, Comparable } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java index 4248b6d9208..ab42ea91880 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java @@ -4,10 +4,12 @@ package io.deephaven.engine.table.impl.select; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; import io.deephaven.engine.table.impl.DependencyStreamProvider; import io.deephaven.util.annotations.TestUseOnly; +import org.jetbrains.annotations.NotNull; import java.util.*; import java.util.stream.Stream; @@ -56,9 +58,11 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { for (WhereFilter filter : componentFilters) { - filter.init(tableDefinition); + filter.init(tableDefinition, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index 3b4f3040ca3..39147a185ba 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Context; import io.deephaven.engine.table.SharedContext; @@ -14,11 +15,11 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.util.codegen.CodeGenerator; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.ColumnSource; import io.deephaven.chunk.*; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; @@ -34,6 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -44,7 +49,7 @@ public class ConditionFilter extends AbstractConditionFilter { public static final int CHUNK_SIZE = 4096; - private Class filterKernelClass = null; + private Future> filterKernelClass = null; private List>> usedInputs; // that is columns and special variables private String classBody; private Filter filter = null; @@ -378,46 +383,53 @@ private static String toTitleCase(String input) { @Override protected void generateFilterCode( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final StringBuilder classBody = getClassBody(tableDefinition, timeConversionResult, result); - if (classBody == null) + if (classBody == null) { return; - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(formula)) { - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - for (String usedColumn : usedColumns) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (String usedColumn : usedColumnArrays) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (final QueryScopeParam param : params) { - addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); - } - - filterKernelClass = ExecutionContext.getContext().getQueryCompiler() - .compile("GeneratedFilterKernel", this.classBody = classBody.toString(), - QueryCompiler.FORMULA_PREFIX, QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); } + + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + for (String usedColumn : usedColumns) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (String usedColumn : usedColumnArrays) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (final QueryScopeParam param : params) { + addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); + } + + this.classBody = classBody.toString(); + + filterKernelClass = compilationProcessor.submit(QueryCompilerRequest.builder() + .description("Filter Expression: " + formula) + .className("GeneratedFilterKernel") + .classBody(this.classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()); } @Nullable private StringBuilder getClassBody( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result) { if (filterKernelClass != null) { return null; } @@ -577,15 +589,23 @@ private StringBuilder getClassBody( protected Filter getFilter(Table table, RowSet fullSet) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { if (filter == null) { - final FilterKernel filterKernel = (FilterKernel) filterKernelClass - .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) - .newInstance(table, fullSet, (Object) params); - final String[] columnNames = usedInputs.stream() - .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) - .toArray(String[]::new); - filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); - // note this filter is not valid for use in other contexts, as it captures references from the source table - filterValidForCopy = false; + try { + final FilterKernel filterKernel = (FilterKernel) filterKernelClass + .get(0, TimeUnit.SECONDS) + .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) + .newInstance(table, fullSet, (Object) params); + final String[] columnNames = usedInputs.stream() + .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) + .toArray(String[]::new); + filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); + // note this filter is not valid for use in other contexts, as it captures references from the source + // table + filterValidForCopy = false; + } catch (TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (InterruptedException | ExecutionException e) { + throw new FormulaCompilationException("Formula compilation error for: " + formula, e); + } } return filter; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index 4a807eb2c79..6dce3731d77 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -3,18 +3,20 @@ */ package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.Pair; import io.deephaven.chunk.ChunkType; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.engine.table.impl.select.codegen.JavaKernelBuilder; import io.deephaven.engine.table.impl.select.codegen.RichType; @@ -31,7 +33,7 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.type.TypeUtils; import io.deephaven.vector.ObjectVector; import io.deephaven.vector.Vector; @@ -48,6 +50,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Function; @@ -182,7 +185,9 @@ public static Class getVectorType(Class declaredType) { } @Override - public List initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (formulaFactory != null) { validateColumnDefinition(columnDefinitionMap); return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; @@ -209,18 +214,22 @@ public List initDef(Map> columnDefinitionMap formulaString = result.getConvertedExpression(); // check if this is a column to be created with a Python vectorizable function - checkAndInitializeVectorization(columnDefinitionMap); + checkAndInitializeVectorization(columnDefinitionMap, compilationRequestProcessor); } catch (Exception e) { throw new FormulaCompilationException("Formula compilation error for: " + formulaString, e); } - formulaFactory = useKernelFormulasProperty - ? createKernelFormulaFactory(getFormulaKernelFactory()) - : createFormulaFactory(); + if (useKernelFormulasProperty) { + formulaFactory = createKernelFormulaFactory(getFormulaKernelFactory(compilationRequestProcessor)); + } else { + compileFormula(compilationRequestProcessor); + } return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; } - private void checkAndInitializeVectorization(Map> columnDefinitionMap) { + private void checkAndInitializeVectorization( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection SuspiciousToArrayCall final PyCallableWrapperJpyImpl[] cws = Arrays.stream(params) .filter(p -> p.getValue() instanceof PyCallableWrapperJpyImpl) @@ -241,7 +250,7 @@ private void checkAndInitializeVectorization(Map> co pyCallableWrapper.getReturnType(), this.analyzedFormula.sourceDescriptor.sources, argumentsChunked, true)); - formulaColumnPython.initDef(columnDefinitionMap); + formulaColumnPython.initDef(columnDefinitionMap, compilationRequestProcessor); } } @@ -705,11 +714,13 @@ protected FormulaSourceDescriptor getSourceDescriptor() { return analyzedFormula.sourceDescriptor; } - protected FormulaKernelFactory getFormulaKernelFactory() { - return invokeKernelBuilder().formulaKernelFactory; + protected CompletionStageFuture getFormulaKernelFactory( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + return invokeKernelBuilder(compilationRequestProcessor).thenApply(result -> result.formulaKernelFactory); } - private JavaKernelBuilder.Result invokeKernelBuilder() { + private CompletionStageFuture invokeKernelBuilder( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final FormulaAnalyzer.Result af = analyzedFormula; final FormulaSourceDescriptor sd = af.sourceDescriptor; final Map columnDict = makeNameToRichTypeDict(sd.sources, columnDefinitions); @@ -722,8 +733,14 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { for (final String p : sd.params) { paramDict.put(p, allParamDict.get(p)); } - return JavaKernelBuilder.create(af.cookedFormulaString, sd.returnType, af.timeInstanceVariables, columnDict, - arrayDict, paramDict); + return JavaKernelBuilder.create( + af.cookedFormulaString, + sd.returnType, + af.timeInstanceVariables, + columnDict, + arrayDict, + paramDict, + compilationRequestProcessor); } /** @@ -731,7 +748,11 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { */ @NotNull String generateKernelClassBody() { - return invokeKernelBuilder().classBody; + try { + return invokeKernelBuilder(QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE).get().classBody; + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Failed to compile formula: ", e); + } } @Override @@ -758,48 +779,46 @@ public Pair>> getFormulaShiftColPair() { return formulaShiftColPair; } - private FormulaFactory createFormulaFactory() { - final String classBody = generateClassBody(); + private void compileFormula(@NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final String what = "Compile regular formula: " + formulaString; - final Class clazz = compileFormula(what, classBody, "Formula"); - try { - return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + what, e); - } - } + final String className = "Formula"; + final String classBody = generateClassBody(); - @SuppressWarnings("SameParameterValue") - private Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: what is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + visitFormulaParameters(null, + csp -> { + addParamClass.accept(csp.type); + addParamClass.accept(csp.columnDefinition.getComponentType()); + return null; + }, + cap -> { + addParamClass.accept(cap.dataType); + addParamClass.accept(cap.columnDefinition.getComponentType()); + return null; + }, + p -> { + addParamClass.accept(p.type); + return null; + }); - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - visitFormulaParameters(null, - csp -> { - addParamClass.accept(csp.type); - addParamClass.accept(csp.columnDefinition.getComponentType()); - return null; - }, - cap -> { - addParamClass.accept(cap.dataType); - addParamClass.accept(cap.columnDefinition.getComponentType()); - return null; - }, - p -> { - addParamClass.accept(p.type); - return null; - }); - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX, - QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); - } + formulaFactory = compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description("Formula Expression: " + formulaString) + .className(className) + .classBody(classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()).thenApply(clazz -> { + try { + return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException("Formula compilation error for: " + what, e); + } + }); } private static class IndexParameter { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java index 378336e4d60..1f884622053 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.DoubleComparisons; import io.deephaven.engine.table.impl.chunkfilter.DoubleRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -79,7 +80,9 @@ static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java index 9f1c9288844..3aa65dda77c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java @@ -9,6 +9,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.updategraph.DynamicNode; import io.deephaven.engine.table.ColumnSource; @@ -85,7 +86,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java index 29152e038d5..e037c2ca258 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java @@ -221,7 +221,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java index 397d807c61b..79a7246cc81 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.FloatComparisons; import io.deephaven.engine.table.impl.chunkfilter.FloatRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -74,7 +75,9 @@ static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java index 2dda0e38dda..549af3ddefd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java @@ -8,6 +8,7 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -106,7 +107,9 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -125,6 +128,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java index 3fae7cfe932..37a087a0022 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; import io.deephaven.engine.table.ColumnSource; @@ -37,7 +38,9 @@ public InstantRangeFilter( } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java index 8badf3ed87c..a2762997a85 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.IntRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -68,7 +69,9 @@ static WhereFilter makeIntRangeFilter(String columnName, Condition condition, St } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java index f5fc6a7cdaf..3ecd0f21457 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.LongRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -68,7 +69,9 @@ static WhereFilter makeLongRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 27b7e671973..38aed9d282a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -10,6 +10,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.context.QueryScope; import io.deephaven.time.DateTimeUtils; @@ -116,7 +117,9 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java index 29d77004a69..8c87ba95f27 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java @@ -9,6 +9,7 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; import io.deephaven.engine.table.impl.PrevColumnSource; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -19,7 +20,6 @@ import io.deephaven.engine.table.impl.chunkfillers.ChunkFiller; import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; -import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; import java.util.*; @@ -94,7 +94,9 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { NoSuchColumnException.throwIf(columnDefinitionMap.keySet(), sourceNames); return getColumns(); } @@ -104,6 +106,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return Collections.unmodifiableList(sourceNames); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java index 8a544d80048..31ebd6a006e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java @@ -5,6 +5,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.*; import io.deephaven.engine.rowset.TrackingRowSet; import org.jetbrains.annotations.NotNull; @@ -32,7 +33,9 @@ public List initInputs(final TrackingRowSet rowSet, } @Override - public List initDef(final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { return Collections.emptyList(); } @@ -41,6 +44,11 @@ public Class getReturnedType() { return nvcs.getType(); } + @Override + public Class getReturnedComponentType() { + return nvcs.getComponentType(); + } + @Override public List getColumns() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index caed4eab668..fbd9806ad76 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.RowSet; @@ -129,7 +130,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (filter != null) { return; } @@ -179,7 +182,7 @@ public void init(TableDefinition tableDefinition) { } } - filter.init(tableDefinition); + filter.init(tableDefinition, compilationProcessor); } public static char parseCharFilter(String value) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java index b76c64456d9..7f672a2a86e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.ConvertibleTimeSource; import io.deephaven.engine.table.impl.sources.LocalDateWrapperSource; import io.deephaven.engine.table.impl.sources.LocalTimeWrapperSource; @@ -38,13 +39,13 @@ /** * Allows {@link ColumnSource} reinterpretation via view-type ({@link Table#view} and {@link Table#updateView}) * {@link Table} operations. - * + *

* TODO: If we come up with other valid, useful reinterpretations, it would be trivial to create a general purpose * syntax for use in view()/updateView() column expressions. - * + *

* The syntax I have in mind is: "<ColumnNameB>=<ColumnNameA>.as(<ClassName>)" * "<ColumnName>.as(<ClassName>)" - * + *

* Making this work would consist of any one of: 1. Adding a V1 version and updating SelectColumnFactory and * SelectColumnAdaptor 2. Adding the appropriate if-regex-matches to realColumn selection in V2 SwitchColumn 3. Creating * a V2-native SelectColumnFactory @@ -151,7 +152,9 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -169,6 +172,12 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + // we don't support reinterpretting column types with components + return null; + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java index dac007e0554..b31f73aa818 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java @@ -9,6 +9,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import org.jetbrains.annotations.NotNull; @@ -45,7 +46,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index 31295a0e0ea..5650d8da788 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -12,6 +12,7 @@ import io.deephaven.api.expression.Method; import io.deephaven.api.filter.Filter; import io.deephaven.api.literal.Literal; +import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.ColumnDefinition; @@ -19,6 +20,8 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; import java.util.Arrays; @@ -71,7 +74,8 @@ static Collection copyFrom(Collection selectColumns) List initInputs(TrackingRowSet rowSet, Map> columnsOfInterest); /** - * Initialize any internal column definitions from the provided initial. + * Initialize any internal column definitions from the provided initial. Any formulae will be compiled immediately + * using the {@link QueryCompiler} in the current {@link ExecutionContext}. * * @param columnDefinitionMap the starting set of column definitions; valid for this call only * @@ -80,7 +84,26 @@ static Collection copyFrom(Collection selectColumns) * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. */ - List initDef(Map> columnDefinitionMap); + @FinalDefault + default List initDef(@NotNull Map> columnDefinitionMap) { + return initDef(columnDefinitionMap, QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); + } + + /** + * Initialize any internal column definitions from the provided initial. A compilation request consumer is provided + * to allow for deferred compilation of expressions that belong to the same query. + * + * @param columnDefinitionMap the starting set of column definitions; valid for this call only + * @param compilationRequestProcessor a consumer to submit compilation requests; valid for this call only + * + * @return a list of columns on which the result of this is dependent + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. + * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. + */ + List initDef( + @NotNull Map> columnDefinitionMap, + @NotNull QueryCompilerRequestProcessor compilationRequestProcessor); /** * Get the data type stored in the resultant column. @@ -89,6 +112,13 @@ static Collection copyFrom(Collection selectColumns) */ Class getReturnedType(); + /** + * Get the data component type stored in the resultant column. + * + * @return the component type + */ + Class getReturnedComponentType(); + /** * Get a list of the names of columns used in this SelectColumn. Behavior is undefined if none of the init* methods * have been called yet. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java index 234123185cf..ce3c0e5ce69 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ShortRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -68,7 +69,9 @@ static WhereFilter makeShortRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java index 2749780468e..610318c7329 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -34,7 +35,9 @@ public static SingleSidedComparableRangeFilter makeForTest(String columnName, Co } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java index bebb014aabd..fb8f3de82f5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java @@ -7,6 +7,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.NoSuchColumnException; @@ -61,7 +62,9 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { sourceDefinition = columnDefinitionMap.get(sourceName); if (sourceDefinition == null) { throw new NoSuchColumnException(columnDefinitionMap.keySet(), sourceName); @@ -78,6 +81,15 @@ public Class getReturnedType() { return sourceColumn.getType(); } + @Override + public Class getReturnedComponentType() { + // Try to be a little flexible, depending on whether initInputs or initDef was called. + if (sourceDefinition != null) { + return sourceDefinition.getComponentType(); + } + return sourceColumn.getComponentType(); + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java index d2483b6433c..37c078fbabf 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java @@ -8,6 +8,7 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.python.FormulaColumnPython; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; @@ -45,7 +46,9 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (realColumn == null) { if (columnDefinitionMap.get(expression) != null) { realColumn = new SourceColumn(expression, columnName); @@ -53,7 +56,7 @@ public List initDef(Map> columnDefinitionMap realColumn = FormulaColumn.createFormulaColumn(columnName, expression, parser); } } - List usedColumns = realColumn.initDef(columnDefinitionMap); + List usedColumns = realColumn.initDef(columnDefinitionMap, compilationRequestProcessor); if (realColumn instanceof DhFormulaColumn) { FormulaColumnPython formulaColumnPython = ((DhFormulaColumn) realColumn).getFormulaColumnPython(); realColumn = formulaColumnPython != null ? formulaColumnPython : realColumn; @@ -66,6 +69,11 @@ public Class getReturnedType() { return getRealColumn().getReturnedType(); } + @Override + public Class getReturnedComponentType() { + return getRealColumn().getReturnedComponentType(); + } + @Override public List getColumns() { return getRealColumn().getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java index 3a736ff098f..174b1e600e5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.time.DateTimeUtils; @@ -55,7 +56,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index e62154dc016..41d5e0018f0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -11,6 +11,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.util.annotations.FinalDefault; @@ -103,7 +104,22 @@ interface RecomputeListener { * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. */ - void init(TableDefinition tableDefinition); + @FinalDefault + default void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); + } + + /** + * Initialize this select filter given the table definition + * + * @param tableDefinition the definition of the table that will be filtered + * @param compilationProcessor the processor to use for compilation + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. + */ + void init( + @NotNull TableDefinition tableDefinition, + @NotNull QueryCompilerRequestProcessor compilationProcessor); /** * Validate that this {@code WhereFilter} is safe to use in the context of the provided sourceTable. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java index 3dbe26c1048..a1eda9e050b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.DependencyStreamProvider; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.util.annotations.VisibleForTesting; import org.jetbrains.annotations.NotNull; @@ -54,8 +55,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { - filter.init(tableDefinition); + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + filter.init(tableDefinition, compilationProcessor); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java index cadbe1bb488..cbe462162de 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter.ObjectChunkFilter; import org.jetbrains.annotations.NotNull; @@ -43,7 +44,9 @@ private WhereFilterPatternImpl(FilterPattern filterPattern) { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final String columnName = columnName(); final ColumnDefinition column = tableDefinition.getColumn(columnName); if (column == null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java index 13afd3d00a2..09c89243f0f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import org.jetbrains.annotations.NotNull; import java.util.Collections; @@ -33,7 +34,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java index f751407b8dc..20ea139c2b1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java @@ -59,16 +59,6 @@ final Map> getColumnSourcesRecurse(GetMode mode) { return result; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - for (Map.Entry> entry : sources.entrySet()) { - final String name = entry.getKey(); - final ColumnSource cs = entry.getValue(); - final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); - columnDefinitions.put(name, cd); - } - } - @Override public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java index cc37e773af2..9a126b15698 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java @@ -36,14 +36,6 @@ public abstract class DependencyLayerBase extends SelectAndViewAnalyzer { this.myModifiedColumnSet = mcsBuilder; } - - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - final ColumnDefinition cd = - ColumnDefinition.fromGenericType(name, columnSource.getType(), columnSource.getComponentType()); - columnDefinitions.put(name, cd); - } - @Override void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { // Later-defined columns override earlier-defined columns. So we satisfy column dependencies "on the way diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java index 27ca97af1b6..fea85388dbb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java @@ -164,11 +164,6 @@ public SelectAndViewAnalyzer getInner() { return inner; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { rowRedirection.startTrackingPrevValues(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index d2972981e17..ea1c45ffa39 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -12,6 +12,7 @@ import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; @@ -47,9 +48,19 @@ public enum Mode { public static void initializeSelectColumns( final Map> parentColumnMap, final SelectColumn[] selectColumns) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + initializeSelectColumns(parentColumnMap, selectColumns, compilationProcessor); + compilationProcessor.compile(); + } + + public static void initializeSelectColumns( + final Map> parentColumnMap, + final SelectColumn[] selectColumns, + final QueryCompilerRequestProcessor compilationProcessor) { final Map> targetColumnMap = new HashMap<>(parentColumnMap); for (SelectColumn column : selectColumns) { - column.initDef(targetColumnMap); + column.initDef(targetColumnMap, compilationProcessor); final ColumnDefinition columnDefinition = ColumnDefinition.fromGenericType(column.getName(), column.getReturnedType()); targetColumnMap.put(column.getName(), columnDefinition); @@ -89,30 +100,62 @@ public static SelectAndViewAnalyzerWrapper create( rowRedirection = null; } - final TrackingRowSet originalRowSet = rowSet; - boolean flatResult = rowSet.isFlat(); - // if we preserve a column, we set this to false - boolean flattenedResult = !flatResult - && allowInternalFlatten - && (columnSources.isEmpty() || !publishTheseSources) - && mode == Mode.SELECT_STATIC; - int numberOfInternallyFlattenedColumns = 0; - List processedCols = new LinkedList<>(); List remainingCols = null; FormulaColumn shiftColumn = null; boolean shiftColumnHasPositiveOffset = false; final HashSet resultColumns = new HashSet<>(); - final HashMap> resultAlias = new HashMap<>(); + + // First pass to initialize all columns and to compile formulas. + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + for (Map.Entry> entry : columnSources.entrySet()) { + final String name = entry.getKey(); + final ColumnSource cs = entry.getValue(); + final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); + columnDefinitions.put(name, cd); + } + for (final SelectColumn sc : selectColumns) { if (remainingCols != null) { remainingCols.add(sc); continue; } - analyzer.updateColumnDefinitionsFromTopLayer(columnDefinitions); - sc.initDef(columnDefinitions); + sc.initDef(columnDefinitions, compilationProcessor); + final ColumnDefinition cd = ColumnDefinition.fromGenericType( + sc.getName(), sc.getReturnedType(), sc.getReturnedComponentType()); + columnDefinitions.put(sc.getName(), cd); + + if (useShiftedColumns && hasConstantArrayAccess(sc)) { + remainingCols = new LinkedList<>(); + shiftColumn = sc instanceof FormulaColumn + ? (FormulaColumn) sc + : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); + shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); + continue; + } + + processedCols.add(sc); + } + + // compile all formulas at once + compilationProcessor.compile(); + + // Second pass builds the analyzer and destination columns + final TrackingRowSet originalRowSet = rowSet; + boolean flatResult = rowSet.isFlat(); + // if we preserve a column, we set this to false + boolean flattenedResult = !flatResult + && allowInternalFlatten + && (columnSources.isEmpty() || !publishTheseSources) + && mode == Mode.SELECT_STATIC; + int numberOfInternallyFlattenedColumns = 0; + + final HashMap> resultAlias = new HashMap<>(); + for (final SelectColumn sc : processedCols) { + sc.initInputs(rowSet, analyzer.getAllColumnSources()); // When flattening the result, intermediate columns generate results in position space. When we discover @@ -138,12 +181,8 @@ public static SelectAndViewAnalyzerWrapper create( final ModifiedColumnSet mcsBuilder = new ModifiedColumnSet(parentMcs); if (useShiftedColumns && hasConstantArrayAccess(sc)) { - remainingCols = new LinkedList<>(); - shiftColumn = sc instanceof FormulaColumn - ? (FormulaColumn) sc - : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); - shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); - continue; + // we use the first shifted column to split between processed columns and remaining columns + throw new IllegalStateException("Found ShiftedColumn in processed column list"); } // shifted columns appear to not be safe for refresh, so we do not validate them until they are rewritten @@ -152,8 +191,6 @@ public static SelectAndViewAnalyzerWrapper create( sc.validateSafeForRefresh(sourceTable); } - processedCols.add(sc); - if (hasConstantValue(sc)) { final WritableColumnSource constViewSource = SingleValueColumnSource.getSingleValueColumnSource(sc.getReturnedType()); @@ -177,8 +214,7 @@ public static SelectAndViewAnalyzerWrapper create( if (!sourceIsNew) { if (numberOfInternallyFlattenedColumns > 0) { // we must preserve this column, but have already created an analyzer for the internally - // flattened - // column, therefore must start over without permitting internal flattening + // flattened column, therefore must start over without permitting internal flattening return create(sourceTable, mode, columnSources, originalRowSet, parentMcs, publishTheseSources, useShiftedColumns, false, selectColumns); } else { @@ -265,6 +301,7 @@ public static SelectAndViewAnalyzerWrapper create( throw new UnsupportedOperationException("Unsupported case " + mode); } } + return new SelectAndViewAnalyzerWrapper(analyzer, shiftColumn, shiftColumnHasPositiveOffset, remainingCols, processedCols); } @@ -522,8 +559,6 @@ public final Map calcEffects(boolean forcePublishAllResources) public abstract SelectAndViewAnalyzer getInner(); - public abstract void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions); - public abstract void startTrackingPrev(); /** @@ -562,8 +597,8 @@ public boolean alreadyFlattenedSources() { abstract public boolean allowCrossColumnParallelization(); /** - * A class that handles the completion of one select column. The handlers are chained together so that when a column - * completes all of the downstream dependencies may execute. + * A class that handles the completion of one select column. The handlers are chained together; all downstream + * dependencies may execute when a column completes. */ public static abstract class SelectLayerCompletionHandler { /** @@ -591,7 +626,7 @@ public static abstract class SelectLayerCompletionHandler { * Create the final completion handler, which has no next handler. * * @param requiredColumns the columns required for this handler to fire - * @param completedColumns the set of completed columns, shared with all of the other handlers + * @param completedColumns the set of completed columns, shared with all the other handlers */ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColumns) { this.requiredColumns = requiredColumns; @@ -601,9 +636,9 @@ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColu /** * Called when a single column is completed. - * + *

* If we are ready, then we call {@link #onAllRequiredColumnsCompleted()}. - * + *

* We may not be ready, but other columns downstream of us may be ready, so they are also notified (the * nextHandler). * @@ -639,7 +674,7 @@ protected void onError(Exception error) { } /** - * Called when all of the required columns are completed. + * Called when all required columns are completed. */ protected abstract void onAllRequiredColumnsCompleted(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java index 1ed084934df..b94fea7abf8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java @@ -117,11 +117,6 @@ int getLayerIndexFor(String column) { return inner.getLayerIndexFor(column); } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { throw new UnsupportedOperationException("StaticFlattenLayer is used in only non-refreshing scenarios"); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java index 7eedb5af3f5..155400a9b9e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java @@ -5,10 +5,11 @@ import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.util.SafeCloseable; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.context.QueryScopeParam; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.DhFormulaColumn; import io.deephaven.engine.table.impl.select.FormulaCompilationException; @@ -30,19 +31,33 @@ public class JavaKernelBuilder { private static final String FORMULA_KERNEL_FACTORY_NAME = "__FORMULA_KERNEL_FACTORY"; - public static Result create(String cookedFormulaString, Class returnedType, String timeInstanceVariables, - Map columns, Map> arrays, Map> params) { - final JavaKernelBuilder jkf = new JavaKernelBuilder(cookedFormulaString, returnedType, timeInstanceVariables, - columns, arrays, params); + public static CompletionStageFuture create( + @NotNull final String cookedFormulaString, + @NotNull final Class returnedType, + @NotNull final String timeInstanceVariables, + @NotNull final Map columns, + @NotNull final Map> arrays, + @NotNull final Map> params, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + final JavaKernelBuilder jkf = new JavaKernelBuilder( + cookedFormulaString, returnedType, timeInstanceVariables, columns, arrays, params); final String classBody = jkf.generateKernelClassBody(); - final Class clazz = compileFormula(cookedFormulaString, classBody, "Formula"); - final FormulaKernelFactory fkf; - try { - fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + cookedFormulaString, e); - } - return new Result(classBody, clazz, fkf); + + return compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description(cookedFormulaString) + .className("Formula") + .classBody(classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()).thenApply(clazz -> { + final FormulaKernelFactory fkf; + try { + fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException( + "Formula compilation error for: " + cookedFormulaString, e); + } + return new Result(classBody, clazz, fkf); + }); } private final String cookedFormulaString; @@ -257,23 +272,16 @@ private List visitFormulaParameters( return results; } - @SuppressWarnings("SameParameterValue") - private static Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: formulaString is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX); - } - } - public static class Result { public final String classBody; public final Class clazz; public final FormulaKernelFactory formulaKernelFactory; - public Result(String classBody, Class clazz, FormulaKernelFactory formulaKernelFactory) { + public Result( + @NotNull final String classBody, + @NotNull final Class clazz, + @NotNull final FormulaKernelFactory formulaKernelFactory) { this.classBody = classBody; this.clazz = clazz; this.formulaKernelFactory = formulaKernelFactory; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java index cc84b09739d..cb93f872e8b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java @@ -5,12 +5,16 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.context.QueryScopeParam; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.select.AbstractFormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.formula.FormulaKernel; import io.deephaven.engine.table.impl.select.formula.FormulaKernelFactory; import io.deephaven.engine.table.impl.select.formula.FormulaSourceDescriptor; +import org.jetbrains.annotations.NotNull; import java.util.LinkedHashSet; import java.util.List; @@ -39,13 +43,15 @@ private FormulaColumnPython(String columnName, } @Override - public final List initDef(Map> columnDefinitionMap) { + public final List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (formulaFactory != null) { validateColumnDefinition(columnDefinitionMap); } else { returnedType = dcf.getReturnedType(); applyUsedVariables(columnDefinitionMap, new LinkedHashSet<>(dcf.getColumnNames()), Map.of()); - formulaFactory = createKernelFormulaFactory(this); + formulaFactory = createKernelFormulaFactory(CompletionStageFuture.completedFuture(this)); } return usedColumns; diff --git a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java index 6991153b155..89132e6f790 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java @@ -5,6 +5,8 @@ import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import java.util.*; import java.util.function.Supplier; @@ -20,7 +22,7 @@ public static Supplier compileSimpleFunction(final Class res public static Supplier compileSimpleStatement(final Class resultType, final String code, final String... imports) { - final List importClasses = new ArrayList<>(); + final List> importClasses = new ArrayList<>(); for (final String importString : imports) { try { importClasses.add(Class.forName(importString)); @@ -33,14 +35,14 @@ public static Supplier compileSimpleStatement(final Class re } public static Supplier compileSimpleFunction(final Class resultType, final String code, - final Collection imports, final Collection staticImports) { + final Collection> imports, final Collection> staticImports) { final StringBuilder classBody = new StringBuilder(); classBody.append("import ").append(resultType.getName()).append(";\n"); - for (final Class im : imports) { + for (final Class im : imports) { classBody.append("import ").append(im.getName()).append(";\n"); } - for (final Class sim : staticImports) { + for (final Class sim : staticImports) { classBody.append("import static ").append(sim.getName()).append(".*;\n"); } @@ -52,8 +54,13 @@ public static Supplier compileSimpleFunction(final Class res classBody.append(" }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Simple Function: " + code) + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()); try { // noinspection unchecked @@ -63,7 +70,7 @@ public static Supplier compileSimpleFunction(final Class res } } - public static Class getClassThroughCompilation(final String object) { + public static Class getClassThroughCompilation(final String object) { final StringBuilder classBody = new StringBuilder(); classBody.append("public class $CLASSNAME$ implements ").append(Supplier.class.getCanonicalName()) .append("{ \n"); @@ -71,12 +78,17 @@ public static Class getClassThroughCompilation(final String object) { classBody.append(" public Class get() { return ").append(object).append(".class; }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Formula: return " + object + ".class") + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()); try { // noinspection unchecked - return ((Supplier) partitionClass.newInstance()).get(); + return ((Supplier>) partitionClass.newInstance()).get(); } catch (InstantiationException | IllegalAccessException e) { throw new RuntimeException("Could not instantiate function.", e); } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java index d60ba8f6795..0db8cda67ae 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java @@ -54,7 +54,9 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull TableDefinition tableDefinition, + @NotNull QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java index fea7cee27b1..9c9971aff15 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java @@ -18,7 +18,7 @@ public void testNotString() { DynamicCompileUtils.compileSimpleFunction(String.class, "return 7"); TestCase.fail("Should never have reached this statement."); } catch (RuntimeException e) { - TestCase.assertTrue(e.getMessage().contains("int cannot be converted to String")); + TestCase.assertTrue(e.getMessage().contains("int cannot be converted to java.lang.String")); } } }