From 2511d4ea7f9c387546a5c6da832e9306e9cc71cf Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 30 Jul 2024 15:58:06 +0800 Subject: [PATCH 1/5] [VL] Recover broken memory-trace option spark.gluten.memory.backtrace.allocation --- cpp/core/jni/JniWrapper.cc | 14 +++++++++++--- cpp/core/memory/AllocationListener.cc | 2 -- cpp/core/memory/AllocationListener.h | 2 -- cpp/velox/compute/VeloxBackend.cc | 3 --- cpp/velox/config/VeloxConfig.h | 3 --- .../scala/org/apache/gluten/GlutenConfig.scala | 11 +++++++++++ 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 3d6da31c7e75..fa25a644b3c4 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -215,6 +215,10 @@ void JNI_OnUnload(JavaVM* vm, void* reserved) { gluten::getJniCommonState()->close(); } +namespace { +const std::string kBacktraceAllocation = "spark.gluten.memory.backtrace.allocation"; +} + JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_createRuntime( // NOLINT JNIEnv* env, jclass, @@ -226,13 +230,17 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_createRunt if (env->GetJavaVM(&vm) != JNI_OK) { throw gluten::GlutenException("Unable to get JavaVM instance"); } - + auto safeArray = gluten::getByteArrayElementsSafe(env, sessionConf); + auto sparkConf = gluten::parseConfMap(env, safeArray.elems(), safeArray.length()); auto backendType = jStringToCString(env, jbackendType); + std::unique_ptr listener = std::make_unique(vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod); + bool backtrace = std::stoi(sparkConf.at(kBacktraceAllocation)); + if (backtrace) { + listener = std::make_unique(std::move(listener)); + } - auto safeArray = gluten::getByteArrayElementsSafe(env, sessionConf); - auto sparkConf = gluten::parseConfMap(env, safeArray.elems(), safeArray.length()); auto runtime = gluten::Runtime::create(backendType, std::move(listener), sparkConf); return reinterpret_cast(runtime); JNI_METHOD_END(kInvalidObjectHandle) diff --git a/cpp/core/memory/AllocationListener.cc b/cpp/core/memory/AllocationListener.cc index 2c876e9f19f7..5cbeeb6bd5c9 100644 --- a/cpp/core/memory/AllocationListener.cc +++ b/cpp/core/memory/AllocationListener.cc @@ -19,8 +19,6 @@ namespace gluten { -bool backtrace_allocation = false; - class NoopAllocationListener : public gluten::AllocationListener { public: void allocationChanged(int64_t diff) override { diff --git a/cpp/core/memory/AllocationListener.h b/cpp/core/memory/AllocationListener.h index 41797641fe14..1751b6112ae2 100644 --- a/cpp/core/memory/AllocationListener.h +++ b/cpp/core/memory/AllocationListener.h @@ -23,8 +23,6 @@ namespace gluten { -extern bool backtrace_allocation; - class AllocationListener { public: static std::unique_ptr noop(); diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index a3658faa3a18..2dad6adf2e70 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -104,9 +104,6 @@ void VeloxBackend::init(const std::unordered_map& conf FLAGS_gluten_velox_aysnc_timeout_on_task_stopping = backendConf_->get(kVeloxAsyncTimeoutOnTaskStopping, kVeloxAsyncTimeoutOnTaskStoppingDefault); - // Set backtrace_allocation - gluten::backtrace_allocation = backendConf_->get(kBacktraceAllocation, false); - // Setup and register. velox::filesystems::registerLocalFileSystem(); initJolFilesystem(); diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h index 65c7cb61d94d..792beda96f7d 100644 --- a/cpp/velox/config/VeloxConfig.h +++ b/cpp/velox/config/VeloxConfig.h @@ -104,9 +104,6 @@ const int32_t kVeloxAsyncTimeoutOnTaskStoppingDefault = 30000; // 30s // udf const std::string kVeloxUdfLibraryPaths = "spark.gluten.sql.columnar.backend.velox.internal.udfLibraryPaths"; -// backtrace allocation -const std::string kBacktraceAllocation = "spark.gluten.backtrace.allocation"; - // VeloxShuffleReader print flag. const std::string kVeloxShuffleReaderPrintFlag = "spark.gluten.velox.shuffleReaderPrintFlag"; diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 5547feafe331..d10409e8c956 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -242,6 +242,8 @@ class GlutenConfig(conf: SQLConf) extends Logging { def memoryIsolation: Boolean = conf.getConf(COLUMNAR_MEMORY_ISOLATION) + def memoryBacktraceAllocation: Boolean = conf.getConf(COLUMNAR_MEMORY_BACKTRACE_ALLOCATION) + def numTaskSlotsPerExecutor: Int = { val numSlots = conf.getConf(NUM_TASK_SLOTS_PER_EXECUTOR) assert(numSlots > 0, s"Number of task slot not found. This should not happen.") @@ -647,6 +649,7 @@ object GlutenConfig { SQLConf.LEGACY_SIZE_OF_NULL.key, "spark.io.compression.codec", "spark.sql.decimalOperations.allowPrecisionLoss", + COLUMNAR_MEMORY_BACKTRACE_ALLOCATION.key, COLUMNAR_VELOX_BLOOM_FILTER_EXPECTED_NUM_ITEMS.key, COLUMNAR_VELOX_BLOOM_FILTER_NUM_BITS.key, COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS.key, @@ -1258,6 +1261,14 @@ object GlutenConfig { .booleanConf .createWithDefault(false) + val COLUMNAR_MEMORY_BACKTRACE_ALLOCATION = + buildConf("spark.gluten.memory.backtrace.allocation") + .internal() + .doc("Print backtrace information for large memory allocations. This helps debugging when " + + "Spark OOM happens due to large acquire requests.") + .booleanConf + .createWithDefault(false) + val COLUMNAR_MEMORY_OVER_ACQUIRED_RATIO = buildConf("spark.gluten.memory.overAcquiredMemoryRatio") .internal() From 4f653e88096ac1202fcfa62d63f73186327468b4 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 30 Jul 2024 16:01:48 +0800 Subject: [PATCH 2/5] fixup --- docs/developers/HowTo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developers/HowTo.md b/docs/developers/HowTo.md index a13bf02ebede..5b16c965fe63 100644 --- a/docs/developers/HowTo.md +++ b/docs/developers/HowTo.md @@ -163,7 +163,7 @@ wait to add # How to track the memory exhaust problem -When your gluten spark jobs failed because of OOM, you can track the memory allocation's call stack by configuring `spark.gluten.backtrace.allocation = true`. +When your gluten spark jobs failed because of OOM, you can track the memory allocation's call stack by configuring `spark.gluten.memory.backtrace.allocation = true`. The above configuration will use `BacktraceAllocationListener` wrapping from `SparkAllocationListener` to create `VeloxMemoryManager`. `BacktraceAllocationListener` will check every allocation, if a single allocation bytes exceeds a fixed value or the accumulative allocation bytes exceeds 1/2/3...G, From 73b598bb46a3d9791817c500e511bb61fe2d69db Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 31 Jul 2024 09:15:43 +0800 Subject: [PATCH 3/5] fixup --- .../src/main/scala/org/apache/gluten/GlutenConfig.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index d10409e8c956..4948b7e30d30 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -649,7 +649,6 @@ object GlutenConfig { SQLConf.LEGACY_SIZE_OF_NULL.key, "spark.io.compression.codec", "spark.sql.decimalOperations.allowPrecisionLoss", - COLUMNAR_MEMORY_BACKTRACE_ALLOCATION.key, COLUMNAR_VELOX_BLOOM_FILTER_EXPECTED_NUM_ITEMS.key, COLUMNAR_VELOX_BLOOM_FILTER_NUM_BITS.key, COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS.key, @@ -676,7 +675,10 @@ object GlutenConfig { val keyWithDefault = ImmutableList.of( (SQLConf.CASE_SENSITIVE.key, SQLConf.CASE_SENSITIVE.defaultValueString), - (SQLConf.IGNORE_MISSING_FILES.key, SQLConf.IGNORE_MISSING_FILES.defaultValueString) + (SQLConf.IGNORE_MISSING_FILES.key, SQLConf.IGNORE_MISSING_FILES.defaultValueString), + ( + COLUMNAR_MEMORY_BACKTRACE_ALLOCATION.key, + COLUMNAR_MEMORY_BACKTRACE_ALLOCATION.defaultValueString) ) keyWithDefault.forEach(e => nativeConfMap.put(e._1, conf.getOrElse(e._1, e._2))) From 8cac01b3fa4e76fe3ea8a1de21a7d157f95ed4ab Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 31 Jul 2024 09:20:48 +0800 Subject: [PATCH 4/5] fixup --- .../src/main/scala/org/apache/gluten/GlutenConfig.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 4948b7e30d30..3184a83310bc 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -725,7 +725,9 @@ object GlutenConfig { (AWS_S3_RETRY_MODE.key, AWS_S3_RETRY_MODE.defaultValueString), ( COLUMNAR_VELOX_CONNECTOR_IO_THREADS.key, - conf.getOrElse(GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY, "-1")), + conf.getOrElse( + NUM_TASK_SLOTS_PER_EXECUTOR.key, + NUM_TASK_SLOTS_PER_EXECUTOR.defaultValueString)), (COLUMNAR_SHUFFLE_CODEC.key, ""), (COLUMNAR_SHUFFLE_CODEC_BACKEND.key, ""), ("spark.hadoop.input.connect.timeout", "180000"), From 59b09785f2c9b0d910304b49df6bc6aa7ba96bc3 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Wed, 31 Jul 2024 09:29:22 +0800 Subject: [PATCH 5/5] fixup --- cpp/core/jni/JniWrapper.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index fa25a644b3c4..8964cad0d1b6 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -236,7 +236,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_createRunt std::unique_ptr listener = std::make_unique(vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod); - bool backtrace = std::stoi(sparkConf.at(kBacktraceAllocation)); + bool backtrace = sparkConf.at(kBacktraceAllocation) == "true"; if (backtrace) { listener = std::make_unique(std::move(listener)); }