[VL] Make glog config be session level (apache#4785)

zhanglistar · Feb 29, 2024 · 908b2f6 · 908b2f6
1 parent 7af6d0c
commit 908b2f6
Show file tree

Hide file tree

Showing 8 changed files with 35 additions and 26 deletions.
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
@@ -61,6 +61,13 @@ const std::string kShuffleCompressionCodecBackend = "spark.gluten.sql.columnar.s
 const std::string kQatBackendName = "qat";
 const std::string kIaaBackendName = "iaa";
 
+// Velox conf
+const std::string kGlogVerboseLevel = "spark.gluten.sql.columnar.backend.velox.glogVerboseLevel";
+const uint32_t kGlogVerboseLevelDefault = 0;
+const uint32_t kGlogVerboseLevelMaximum = 99;
+const std::string kGlogSeverityLevel = "spark.gluten.sql.columnar.backend.velox.glogSeverityLevel";
+const uint32_t kGlogSeverityLevelDefault = 1;
+
 std::unordered_map<std::string, std::string>
 parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength);
 

diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc
@@ -60,12 +60,6 @@ const std::string kEnableUserExceptionStacktrace =
     "spark.gluten.sql.columnar.backend.velox.enableUserExceptionStacktrace";
 const bool kEnableUserExceptionStacktraceDefault = true;
 
-const std::string kGlogVerboseLevel = "spark.gluten.sql.columnar.backend.velox.glogVerboseLevel";
-const uint32_t kGlogVerboseLevelDefault = 0;
-
-const std::string kGlogSeverityLevel = "spark.gluten.sql.columnar.backend.velox.glogSeverityLevel";
-const uint32_t kGlogSeverityLevelDefault = 1;
-
 const std::string kEnableSystemExceptionStacktrace =
     "spark.gluten.sql.columnar.backend.velox.enableSystemExceptionStacktrace";
 const bool kEnableSystemExceptionStacktraceDefault = true;
@@ -148,12 +142,14 @@ void VeloxBackend::init(const std::unordered_map<std::string, std::string>& conf
 
   // Init glog and log level.
   if (!veloxcfg->get<bool>(kDebugModeEnabled, false)) {
-    uint32_t vlogLevel = veloxcfg->get<uint32_t>(kGlogVerboseLevel, kGlogVerboseLevelDefault);
-    FLAGS_v = vlogLevel;
-    uint32_t severityLogLevel = veloxcfg->get<uint32_t>(kGlogSeverityLevel, kGlogSeverityLevelDefault);
-    FLAGS_minloglevel = severityLogLevel;
+    FLAGS_v = veloxcfg->get<uint32_t>(kGlogVerboseLevel, kGlogVerboseLevelDefault);
+    FLAGS_minloglevel = veloxcfg->get<uint32_t>(kGlogSeverityLevel, kGlogSeverityLevelDefault);
   } else {
-    FLAGS_v = 99;
+    if (veloxcfg->isValueExists(kGlogVerboseLevel)) {
+      FLAGS_v = veloxcfg->get<uint32_t>(kGlogVerboseLevel, kGlogVerboseLevelDefault);
+    } else {
+      FLAGS_v = kGlogVerboseLevelMaximum;
+    }
   }
   FLAGS_logtostderr = true;
   google::InitGoogleLogging("gluten");

diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc
@@ -37,15 +37,21 @@ using namespace facebook;
 
 namespace gluten {
 
-VeloxRuntime::VeloxRuntime(const std::unordered_map<std::string, std::string>& confMap) : Runtime(confMap) {}
+VeloxRuntime::VeloxRuntime(const std::unordered_map<std::string, std::string>& confMap) : Runtime(confMap) {
+  // Refresh session config.
+  veloxCfg_ = std::make_shared<const facebook::velox::core::MemConfigMutable>(confMap_);
+  debugModeEnabled_ = veloxCfg_->get<bool>(kDebugModeEnabled, false);
+  FLAGS_minloglevel = veloxCfg_->get<uint32_t>(kGlogSeverityLevel, FLAGS_minloglevel);
+  FLAGS_v = veloxCfg_->get<uint32_t>(kGlogVerboseLevel, FLAGS_v);
+}
 
 void VeloxRuntime::parsePlan(
     const uint8_t* data,
     int32_t size,
     SparkTaskInfo taskInfo,
     std::optional<std::string> dumpFile) {
   taskInfo_ = taskInfo;
-  if (debugModeEnabled(confMap_)) {
+  if (debugModeEnabled_) {
     try {
       auto jsonPlan = substraitFromPbToJson("Plan", data, size, dumpFile);
       LOG(INFO) << std::string(50, '#') << " received substrait::Plan:";
@@ -59,7 +65,7 @@ void VeloxRuntime::parsePlan(
 }
 
 void VeloxRuntime::parseSplitInfo(const uint8_t* data, int32_t size, std::optional<std::string> dumpFile) {
-  if (debugModeEnabled(confMap_)) {
+  if (debugModeEnabled_) {
     try {
       auto jsonPlan = substraitFromPbToJson("ReadRel.LocalFiles", data, size, dumpFile);
       LOG(INFO) << std::string(50, '#') << " received substrait::ReadRel.LocalFiles:";
@@ -111,7 +117,7 @@ std::shared_ptr<ResultIterator> VeloxRuntime::createResultIterator(
     const std::string& spillDir,
     const std::vector<std::shared_ptr<ResultIterator>>& inputs,
     const std::unordered_map<std::string, std::string>& sessionConf) {
-  if (debugModeEnabled(confMap_)) {
+  if (debugModeEnabled_) {
     LOG(INFO) << "VeloxRuntime session config:" << printConfig(confMap_);
   }
 

diff --git a/cpp/velox/compute/VeloxRuntime.h b/cpp/velox/compute/VeloxRuntime.h
@@ -120,6 +120,10 @@ class VeloxRuntime final : public Runtime {
     return veloxPlan_;
   }
 
+  bool debugModeEnabled() const {
+    return debugModeEnabled_;
+  }
+
   static void getInfoAndIds(
       const std::unordered_map<facebook::velox::core::PlanNodeId, std::shared_ptr<SplitInfo>>& splitInfoMap,
       const std::unordered_set<facebook::velox::core::PlanNodeId>& leafPlanNodeIds,
@@ -129,6 +133,8 @@ class VeloxRuntime final : public Runtime {
 
  private:
   std::shared_ptr<const facebook::velox::core::PlanNode> veloxPlan_;
+  std::shared_ptr<const facebook::velox::Config> veloxCfg_;
+  bool debugModeEnabled_{false};
 
   std::unordered_map<int32_t, std::shared_ptr<ColumnarBatch>> emptySchemaBatchLoopUp_;
 };

diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc
@@ -23,6 +23,7 @@
 #include <exception>
 #include "JniUdf.h"
 #include "compute/VeloxBackend.h"
+#include "compute/VeloxRuntime.h"
 #include "config/GlutenConfig.h"
 #include "jni/JniError.h"
 #include "jni/JniFileSystem.h"
@@ -94,7 +95,8 @@ Java_io_glutenproject_vectorized_PlanEvaluatorJniWrapper_nativeValidateWithFailu
   auto safeArray = gluten::getByteArrayElementsSafe(env, planArray);
   auto planData = safeArray.elems();
   auto planSize = env->GetArrayLength(planArray);
-  if (gluten::debugModeEnabled(ctx->getConfMap())) {
+  auto runtime = dynamic_cast<gluten::VeloxRuntime*>(ctx);
+  if (runtime->debugModeEnabled()) {
     try {
       auto jsonPlan = gluten::substraitFromPbToJson("Plan", planData, planSize, std::nullopt);
       LOG(INFO) << std::string(50, '#') << " received substrait::Plan: for validation";

diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc
@@ -52,12 +52,6 @@ std::string getConfigValue(
   return got->second;
 }
 
-bool debugModeEnabled(const std::unordered_map<std::string, std::string>& confMap) {
-  std::shared_ptr<const facebook::velox::Config> veloxCfg =
-      std::make_shared<const facebook::velox::core::MemConfigMutable>(confMap);
-  return veloxCfg->get<bool>(kDebugModeEnabled, false);
-}
-
 std::shared_ptr<facebook::velox::core::MemConfigMutable> getHiveConfig(
     const std::shared_ptr<const facebook::velox::Config>& conf) {
   auto hiveConf = std::make_shared<facebook::velox::core::MemConfigMutable>();

diff --git a/cpp/velox/utils/ConfigExtractor.h b/cpp/velox/utils/ConfigExtractor.h
@@ -33,8 +33,6 @@ std::string getConfigValue(
     const std::string& key,
     const std::optional<std::string>& fallbackValue);
 
-bool debugModeEnabled(const std::unordered_map<std::string, std::string>& confMap);
-
 std::shared_ptr<facebook::velox::core::MemConfigMutable> getHiveConfig(
     const std::shared_ptr<const facebook::velox::Config>& conf);
 

diff --git a/shims/common/src/main/scala/io/glutenproject/GlutenConfig.scala b/shims/common/src/main/scala/io/glutenproject/GlutenConfig.scala
@@ -1195,14 +1195,14 @@ object GlutenConfig {
       .createWithDefault(2)
 
   val COLUMNAR_VELOX_GLOG_VERBOSE_LEVEL =
-    buildStaticConf("spark.gluten.sql.columnar.backend.velox.glogVerboseLevel")
+    buildConf("spark.gluten.sql.columnar.backend.velox.glogVerboseLevel")
       .internal()
       .doc("Set glog verbose level in Velox backend, same as FLAGS_v.")
       .intConf
       .createWithDefault(0)
 
   val COLUMNAR_VELOX_GLOG_SEVERITY_LEVEL =
-    buildStaticConf("spark.gluten.sql.columnar.backend.velox.glogSeverityLevel")
+    buildConf("spark.gluten.sql.columnar.backend.velox.glogSeverityLevel")
       .internal()
       .doc("Set glog severity level in Velox backend, same as FLAGS_minloglevel.")
       .intConf