redact sensitive confs

apache · Aug 12, 2024 · 961c922 · 961c922
1 parent 5d6c6f3
commit 961c922
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 11 deletions.
diff --git a/cpp/core/config/GlutenConfig.cc b/cpp/core/config/GlutenConfig.cc
@@ -20,26 +20,47 @@
 #include "compute/ProtobufUtils.h"
 #include "config.pb.h"
 #include "jni/JniError.h"
+#include <regex>
+#include <optional>
 
 namespace gluten {
+
+const std::string REDACTED_VALUE = "*********(redacted)";
+const std::string REGEX_REDACT_KEY = "spark.gluten.redaction.regex";
+
 std::unordered_map<std::string, std::string>
-parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength) {
-  std::unordered_map<std::string, std::string> sparkConfs;
-  ConfigMap pConfigMap;
-  gluten::parseProtobuf(planData, planDataLength, &pConfigMap);
-  for (const auto& pair : pConfigMap.configs()) {
-    sparkConfs.emplace(pair.first, pair.second);
+  parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength) {
+    std::unordered_map<std::string, std::string> sparkConfs;
+    ConfigMap pConfigMap;
+    gluten::parseProtobuf(planData, planDataLength, &pConfigMap);
+    for (const auto& pair : pConfigMap.configs()) {
+      sparkConfs.emplace(pair.first, pair.second);
   }
-
   return sparkConfs;
 }
 
+std::optional<std::regex> getRedactionRegex(const std::unordered_map<std::string, std::string>& conf) {
+    auto it = conf.find(REGEX_REDACT_KEY);
+    if (it != conf.end()) {
+        return std::regex(it->second);
+    }
+    return std::nullopt;
+}
+
 std::string printConfig(const std::unordered_map<std::string, std::string>& conf) {
   std::ostringstream oss;
   oss << std::endl;
-  for (auto& [k, v] : conf) {
-    oss << " [" << k << ", " << v << "]\n";
+
+    auto redactionRegex = getRedactionRegex(conf);
+
+  for (const auto& [k, v] : conf) {
+    if (redactionRegex && std::regex_match(k, *redactionRegex)) {
+        oss << " [" << k << ", " << REDACTED_VALUE << "]\n";
+    } else {
+        oss << " [" << k << ", " << v << "]\n";
+    }
   }
   return oss.str();
 }
+
 } // namespace gluten
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
@@ -64,6 +64,8 @@ const std::string kShuffleCompressionCodecBackend = "spark.gluten.sql.columnar.s
 const std::string kQatBackendName = "qat";
 const std::string kIaaBackendName = "iaa";
 
+const std::string kRedactionRegex = "spark.gluten.redaction.regex";
+
 std::unordered_map<std::string, std::string>
 parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t planDataLength);
 

diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -622,6 +622,7 @@ object GlutenConfig {
 
   val GLUTEN_COST_EVALUATOR_ENABLED = "spark.gluten.sql.adaptive.costEvaluator.enabled"
 
+  val GLUTEN_REGEX_LOG_REDACTION = "spark.gluten.redaction.regex"
   var ins: GlutenConfig = _
 
   def getConf: GlutenConfig = {
@@ -673,7 +674,8 @@ object GlutenConfig {
       // gcs config
       SPARK_GCS_STORAGE_ROOT_URL,
       SPARK_GCS_AUTH_TYPE,
-      SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE
+      SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE,
+      GLUTEN_REGEX_LOG_REDACTION
     )
     nativeConfMap.putAll(conf.filter(e => keys.contains(e._1)).asJava)
 
@@ -757,7 +759,8 @@ object GlutenConfig {
       GLUTEN_TASK_OFFHEAP_SIZE_IN_BYTES_KEY,
       GLUTEN_OFFHEAP_ENABLED,
       SESSION_LOCAL_TIMEZONE.key,
-      DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key
+      DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key,
+      GLUTEN_REGEX_LOG_REDACTION
     )
     nativeConfMap.putAll(conf.filter(e => keys.contains(e._1)).asJava)