diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java index b8b4138dc8c05..5b2b64909eafb 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java @@ -36,15 +36,12 @@ import scala.Tuple2; import scala.collection.JavaConverters; -public class CHNativeExpressionEvaluator { - private final ExpressionEvaluatorJniWrapper jniWrapper; +public class CHNativeExpressionEvaluator extends ExpressionEvaluatorJniWrapper { - public CHNativeExpressionEvaluator() { - jniWrapper = new ExpressionEvaluatorJniWrapper(); - } + private CHNativeExpressionEvaluator() {} // Used to initialize the native computing. - public void initNative(SparkConf conf) { + public static void initNative(SparkConf conf) { Tuple2[] all = conf.getAll(); Map confMap = Arrays.stream(all).collect(Collectors.toMap(Tuple2::_1, Tuple2::_2)); @@ -55,19 +52,19 @@ public void initNative(SparkConf conf) { // Get the customer config from SparkConf for each backend BackendsApiManager.getTransformerApiInstance().postProcessNativeConfig(nativeConfMap, prefix); - jniWrapper.nativeInitNative(buildNativeConf(nativeConfMap)); + nativeInitNative(buildNativeConf(nativeConfMap)); } - public void finalizeNative() { - jniWrapper.nativeFinalizeNative(); + public static void finalizeNative() { + nativeFinalizeNative(); } // Used to validate the Substrait plan in native compute engine. - public boolean doValidate(byte[] subPlan) { - return jniWrapper.nativeDoValidate(subPlan); + public static boolean doValidate(byte[] subPlan) { + throw new UnsupportedOperationException("doValidate is not supported in Clickhouse Backend"); } - private byte[] buildNativeConf(Map confs) { + private static byte[] buildNativeConf(Map confs) { StringMapNode stringMapNode = ExpressionBuilder.makeStringMap(confs); AdvancedExtensionNode extensionNode = ExtensionBuilder.makeAdvancedExtension( @@ -76,22 +73,21 @@ private byte[] buildNativeConf(Map confs) { return PlanBuilder.makePlan(extensionNode).toProtobuf().toByteArray(); } - private Map getNativeBackendConf() { + private static Map getNativeBackendConf() { return GlutenConfig.getNativeBackendConf( BackendsApiManager.getSettings().getBackendConfigPrefix(), SQLConf.get().getAllConfs()); } // Used by WholeStageTransform to create the native computing pipeline and // return a columnar result iterator. - public BatchIterator createKernelWithBatchIterator( + public static BatchIterator createKernelWithBatchIterator( byte[] wsPlan, byte[][] splitInfo, List iterList, boolean materializeInput) { - long allocId = CHNativeMemoryAllocators.contextInstance().getNativeInstanceId(); long handle = - jniWrapper.nativeCreateKernelWithIterator( - allocId, + nativeCreateKernelWithIterator( + CHNativeMemoryAllocators.contextInstance().getNativeInstanceId(), wsPlan, splitInfo, iterList.toArray(new GeneralInIterator[0]), @@ -101,10 +97,10 @@ public BatchIterator createKernelWithBatchIterator( } // Only for UT. - public BatchIterator createKernelWithBatchIterator( + public static BatchIterator createKernelWithBatchIterator( long allocId, byte[] wsPlan, byte[][] splitInfo, List iterList) { long handle = - jniWrapper.nativeCreateKernelWithIterator( + nativeCreateKernelWithIterator( allocId, wsPlan, splitInfo, @@ -114,7 +110,7 @@ public BatchIterator createKernelWithBatchIterator( return createBatchIterator(handle); } - private BatchIterator createBatchIterator(long nativeHandle) { + private static BatchIterator createBatchIterator(long nativeHandle) { return new BatchIterator(nativeHandle); } } diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ExpressionEvaluatorJniWrapper.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ExpressionEvaluatorJniWrapper.java index 0a008a04b5596..7c67f1008e383 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ExpressionEvaluatorJniWrapper.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/ExpressionEvaluatorJniWrapper.java @@ -24,18 +24,10 @@ public class ExpressionEvaluatorJniWrapper { /** Call initNative to initialize native computing. */ - native void nativeInitNative(byte[] confAsPlan); + static native void nativeInitNative(byte[] confAsPlan); /** Call finalizeNative to finalize native computing. */ - native void nativeFinalizeNative(); - - /** - * Validate the Substrait plan in native compute engine. - * - * @param subPlan the Substrait plan in binary format. - * @return whether the computing of this plan is supported in native. - */ - native boolean nativeDoValidate(byte[] subPlan); + static native void nativeFinalizeNative(); /** * Create a native compute kernel and return a columnar result iterator. @@ -43,18 +35,11 @@ public class ExpressionEvaluatorJniWrapper { * @param allocatorId allocator id * @return iterator instance id */ - public native long nativeCreateKernelWithIterator( + public static native long nativeCreateKernelWithIterator( long allocatorId, byte[] wsPlan, byte[][] splitInfo, GeneralInIterator[] batchItr, byte[] confArray, boolean materializeInput); - - /** - * Closes the projector referenced by nativeHandler. - * - * @param nativeHandler nativeHandler that needs to be closed - */ - native void nativeClose(long nativeHandler); } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 4b9ec739028f6..0c8495c6f67e7 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -76,7 +76,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { } .map(it => new ColumnarNativeIterator(it.asJava).asInstanceOf[GeneralInIterator]) .asJava - new CHNativeExpressionEvaluator().createKernelWithBatchIterator( + CHNativeExpressionEvaluator.createKernelWithBatchIterator( wsPlan, splitInfoByteArray, listIterator, diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala index 0110d085b98c8..69797feb65fb7 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala @@ -83,7 +83,7 @@ class CHListenerApi extends ListenerApi with Logging { val externalSortKey = s"${CHBackendSettings.getBackendConfigPrefix}.runtime_settings" + s".max_bytes_before_external_sort" if (conf.getLong(externalSortKey, -1) < 0) { - if (conf.getBoolean("spark.memory.offHeap.enabled", false)) { + if (conf.getBoolean("spark.memory.offHeap.enabled", defaultValue = false)) { val memSize = JavaUtils.byteStringAsBytes(conf.get("spark.memory.offHeap.size")) if (memSize > 0L) { val cores = conf.getInt("spark.executor.cores", 1).toLong @@ -97,8 +97,7 @@ class CHListenerApi extends ListenerApi with Logging { // Load supported hive/python/scala udfs UDFMappings.loadFromSparkConf(conf) - val initKernel = new CHNativeExpressionEvaluator() - initKernel.initNative(conf) + CHNativeExpressionEvaluator.initNative(conf) // inject backend-specific implementations to override spark classes // FIXME: The following set instances twice in local mode? @@ -110,7 +109,6 @@ class CHListenerApi extends ListenerApi with Logging { private def shutdown(): Unit = { CHBroadcastBuildSideCache.cleanAll() - val kernel = new CHNativeExpressionEvaluator() - kernel.finalizeNative() + CHNativeExpressionEvaluator.finalizeNative() } } diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala index 9e4cbf760974b..09659e42ca838 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala @@ -38,8 +38,7 @@ import org.apache.spark.sql.execution.datasources.v2.V2CommandExec class CHValidatorApi extends ValidatorApi with AdaptiveSparkPlanHelper with Logging { override def doNativeValidateWithFailureReason(plan: PlanNode): NativePlanValidationInfo = { - val validator = new CHNativeExpressionEvaluator() - if (validator.doValidate(plan.toProtobuf.toByteArray)) { + if (CHNativeExpressionEvaluator.doValidate(plan.toProtobuf.toByteArray)) { new NativePlanValidationInfo(1, "") } else { new NativePlanValidationInfo(0, "CH native check failed.") diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala index af512934bc96b..2f11613a275c4 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/execution/NativeFileScanColumnarRDD.scala @@ -49,9 +49,8 @@ class NativeFileScanColumnarRDD( val resIter = GlutenTimeMetric.millis(scanTime) { _ => - val transKernel = new CHNativeExpressionEvaluator() val inBatchIters = new util.ArrayList[GeneralInIterator]() - transKernel.createKernelWithBatchIterator( + CHNativeExpressionEvaluator.createKernelWithBatchIterator( inputPartition.plan, splitInfoByteArray, inBatchIters, diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala index ee0ad8039afc1..3253e04bb36cc 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/metrics/GlutenClickHouseMetricsUTUtils.scala @@ -45,9 +45,8 @@ object GlutenClickHouseMetricsUTUtils { SubstraitPlanPrinterUtil.jsonToSubstraitPlan( substraitPlanJsonStr.replaceAll("basePath", basePath.substring(1))) - val transKernel = new CHNativeExpressionEvaluator() val mockMemoryAllocator = CHNativeMemoryAllocators.contextInstanceForUT() - val resIter = transKernel.createKernelWithBatchIterator( + val resIter = CHNativeExpressionEvaluator.createKernelWithBatchIterator( mockMemoryAllocator.getNativeInstanceId, substraitPlan.toByteArray, new Array[Array[Byte]](0), diff --git a/cpp-ch/local-engine/local_engine_jni.cpp b/cpp-ch/local-engine/local_engine_jni.cpp index d403f8a3d2967..3c3e839a3b29f 100644 --- a/cpp-ch/local-engine/local_engine_jni.cpp +++ b/cpp-ch/local-engine/local_engine_jni.cpp @@ -201,7 +201,7 @@ JNIEXPORT void JNI_OnUnload(JavaVM * vm, void * /*reserved*/) env->DeleteGlobalRef(local_engine::ReservationListenerWrapper::reservation_listener_class); } -JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeInitNative(JNIEnv * env, jobject, jbyteArray conf_plan) +JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeInitNative(JNIEnv * env, jclass, jbyteArray conf_plan) { LOCAL_ENGINE_JNI_METHOD_START const auto conf_plan_a = local_engine::getByteArrayElementsSafe(env, conf_plan); @@ -210,7 +210,7 @@ JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_n LOCAL_ENGINE_JNI_METHOD_END(env, ) } -JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeFinalizeNative(JNIEnv * env) +JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeFinalizeNative(JNIEnv * env, jclass) { LOCAL_ENGINE_JNI_METHOD_START local_engine::BackendFinalizerUtil::finalizeSessionally(); @@ -219,7 +219,7 @@ JNIEXPORT void Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_n JNIEXPORT jlong Java_org_apache_gluten_vectorized_ExpressionEvaluatorJniWrapper_nativeCreateKernelWithIterator( JNIEnv * env, - jobject /*obj*/, + jclass , jlong allocator_id, jbyteArray plan, jobjectArray split_infos,