diff --git a/cpp/core/config/GlutenConfig.cc b/cpp/core/config/GlutenConfig.cc index 824e88b9e554..5674c7e7f0c4 100644 --- a/cpp/core/config/GlutenConfig.cc +++ b/cpp/core/config/GlutenConfig.cc @@ -24,9 +24,8 @@ namespace { -const std::string REGEX_REDACT_KEY = "spark.gluten.redaction.regex"; std::optional getRedactionRegex(const std::unordered_map& conf) { - auto it = conf.find(REGEX_REDACT_KEY); + auto it = conf.find(gluten::kSparkRedactionRegex); if (it != conf.end()) { return std::regex(it->second); } @@ -36,8 +35,6 @@ std::optional getRedactionRegex(const std::unordered_map parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength) { std::unordered_map sparkConfs; @@ -58,7 +55,7 @@ std::string printConfig(const std::unordered_map& conf for (const auto& [k, v] : conf) { if (redactionRegex && std::regex_match(k, *redactionRegex)) { - oss << " [" << k << ", " << REDACTED_VALUE << "]\n"; + oss << " [" << k << ", " << kSparkRedactionString << "]\n"; } else { oss << " [" << k << ", " << v << "]\n"; } diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index 060bbe111265..e8eb65861e83 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -64,6 +64,9 @@ const std::string kShuffleCompressionCodecBackend = "spark.gluten.sql.columnar.s const std::string kQatBackendName = "qat"; const std::string kIaaBackendName = "iaa"; +const std::string kSparkRedactionRegex = "spark.redaction.regex"; +const std::string kSparkRedactionString = "*********(redacted)"; + std::unordered_map parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength); diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala index 435fa04ca714..9c3e24233259 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala @@ -534,6 +534,7 @@ object GlutenConfig { val GLUTEN_ONHEAP_SIZE_KEY = "spark.executor.memory" val GLUTEN_OFFHEAP_SIZE_KEY = "spark.memory.offHeap.size" val GLUTEN_OFFHEAP_ENABLED = "spark.memory.offHeap.enabled" + val SPARK_REDACTION_REGEX = "spark.redaction.regex" // For Soft Affinity Scheduling // Enable Soft Affinity Scheduling, defalut value is false @@ -622,7 +623,6 @@ object GlutenConfig { val GLUTEN_COST_EVALUATOR_ENABLED = "spark.gluten.sql.adaptive.costEvaluator.enabled" - val GLUTEN_REGEX_LOG_REDACTION = "spark.gluten.redaction.regex" var ins: GlutenConfig = _ def getConf: GlutenConfig = { @@ -675,7 +675,7 @@ object GlutenConfig { SPARK_GCS_STORAGE_ROOT_URL, SPARK_GCS_AUTH_TYPE, SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE, - GLUTEN_REGEX_LOG_REDACTION + SPARK_REDACTION_REGEX ) nativeConfMap.putAll(conf.filter(e => keys.contains(e._1)).asJava) @@ -760,7 +760,7 @@ object GlutenConfig { GLUTEN_OFFHEAP_ENABLED, SESSION_LOCAL_TIMEZONE.key, DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key, - GLUTEN_REGEX_LOG_REDACTION + SPARK_REDACTION_REGEX ) nativeConfMap.putAll(conf.filter(e => keys.contains(e._1)).asJava)