From 3b85462d7b35c233a9ca1226509ac2e0550c9e3f Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Fri, 10 Nov 2023 03:27:10 +0000 Subject: [PATCH 1/2] fix uncompressed not set --- cpp/core/jni/JniWrapper.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index e311da0d6177..230f8b42e3b1 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -785,8 +785,8 @@ JNIEXPORT jlong JNICALL Java_io_glutenproject_vectorized_ShuffleWriterJniWrapper shuffleWriterOptions.buffer_size = bufferSize; } + shuffleWriterOptions.compression_type = getCompressionType(env, codecJstr); if (codecJstr != NULL) { - shuffleWriterOptions.compression_type = getCompressionType(env, codecJstr); shuffleWriterOptions.codec_backend = getCodecBackend(env, codecBackendJstr); shuffleWriterOptions.compression_mode = getCompressionMode(env, compressionModeJstr); } From c62b5f791fb7c9dac1f66665b89954dff1089bd5 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Mon, 13 Nov 2023 01:06:36 +0000 Subject: [PATCH 2/2] fix reader --- cpp/core/jni/JniCommon.h | 17 +++++++---------- cpp/core/jni/JniWrapper.cc | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h index 4d4bf5e3fe56..265382aa2d20 100644 --- a/cpp/core/jni/JniCommon.h +++ b/cpp/core/jni/JniCommon.h @@ -170,18 +170,15 @@ static inline arrow::Compression::type getCompressionType(JNIEnv* env, jstring c if (codecJstr == NULL) { return arrow::Compression::UNCOMPRESSED; } - auto codecU = env->GetStringUTFChars(codecJstr, JNI_FALSE); + auto codec = env->GetStringUTFChars(codecJstr, JNI_FALSE); - std::string codecL; - std::transform(codecU, codecU + std::strlen(codecU), std::back_inserter(codecL), ::tolower); + // Convert codec string into lowercase. + std::string codecLower; + std::transform(codec, codec + std::strlen(codec), std::back_inserter(codecLower), ::tolower); + GLUTEN_ASSIGN_OR_THROW(auto compressionType, arrow::util::Codec::GetCompressionType(codecLower)); - GLUTEN_ASSIGN_OR_THROW(auto compression_type, arrow::util::Codec::GetCompressionType(codecL)); - - if (compression_type == arrow::Compression::LZ4) { - compression_type = arrow::Compression::LZ4_FRAME; - } - env->ReleaseStringUTFChars(codecJstr, codecU); - return compression_type; + env->ReleaseStringUTFChars(codecJstr, codec); + return compressionType; } static inline gluten::CodecBackend getCodecBackend(JNIEnv* env, jstring codecJstr) { diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 230f8b42e3b1..189c674f3295 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -991,8 +991,8 @@ JNIEXPORT jlong JNICALL Java_io_glutenproject_vectorized_ShuffleReaderJniWrapper ShuffleReaderOptions options = ShuffleReaderOptions::defaults(); options.ipc_read_options.memory_pool = pool; options.ipc_read_options.use_threads = false; + options.compression_type = getCompressionType(env, compressionType); if (compressionType != nullptr) { - options.compression_type = getCompressionType(env, compressionType); options.codec_backend = getCodecBackend(env, compressionBackend); } std::shared_ptr schema =