address comments

apache · Jun 20, 2024 · 67f3a90 · 67f3a90
1 parent db4a8fd
commit 67f3a90
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc
@@ -39,7 +39,8 @@ const std::string kVeloxS3RetryMode = "spark.gluten.velox.fs.s3a.retry.mode";
 const std::string kVeloxS3RetryModeDefault = "legacy";
 // Connection timeout for AWS s3
 const std::string kVeloxS3ConnectTimeout = "spark.gluten.velox.fs.s3a.connect.timeout";
-const std::string kVeloxS3ConnectTimeoutDefault = "1s";
+// Using default fs.s3a.connection.timeout value in hadoop
+const std::string kVeloxS3ConnectTimeoutDefault = "200s";
 } // namespace
 
 namespace gluten {
@@ -70,9 +71,9 @@ std::shared_ptr<facebook::velox::core::MemConfig> getHiveConfig(std::shared_ptr<
   bool useInstanceCredentials = conf->get<bool>("spark.hadoop.fs.s3a.use.instance.credentials", false);
   std::string iamRole = conf->get<std::string>("spark.hadoop.fs.s3a.iam.role", "");
   std::string iamRoleSessionName = conf->get<std::string>("spark.hadoop.fs.s3a.iam.role.session.name", "");
-  std::string retryMaxAttempts = conf->get<std::string>("spark.hadoop.fs.s3a.retry.limit", "3");
+  std::string retryMaxAttempts = conf->get<std::string>("spark.hadoop.fs.s3a.retry.limit", "20");
   std::string retryMode = conf->get<std::string>(kVeloxS3RetryMode, kVeloxS3RetryModeDefault);
-  std::string maxConnections = conf->get<std::string>("spark.hadoop.fs.s3a.connection.maximum", "96");
+  std::string maxConnections = conf->get<std::string>("spark.hadoop.fs.s3a.connection.maximum", "15");
   std::string connectTimeout = conf->get<std::string>(kVeloxS3ConnectTimeout, kVeloxS3ConnectTimeoutDefault);
 
   std::string awsSdkLogLevel = conf->get<std::string>(kVeloxAwsSdkLogLevel, kVeloxAwsSdkLogLevelDefault);

diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -704,8 +704,8 @@ object GlutenConfig {
       (SPARK_S3_USE_INSTANCE_CREDENTIALS, "false"),
       (SPARK_S3_IAM, ""),
       (SPARK_S3_IAM_SESSION_NAME, ""),
-      (SPARK_S3_RETRY_MAX_ATTEMPTS, "3"),
-      (SPARK_S3_CONNECTION_MAXIMUM, "96"),
+      (SPARK_S3_RETRY_MAX_ATTEMPTS, "20"),
+      (SPARK_S3_CONNECTION_MAXIMUM, "15"),
       (AWS_S3_CONNECT_TIMEOUT.key, AWS_S3_CONNECT_TIMEOUT.defaultValueString),
       (AWS_S3_RETRY_MODE.key, AWS_S3_RETRY_MODE.defaultValueString),
       (
@@ -1951,7 +1951,7 @@ object GlutenConfig {
   val AWS_S3_RETRY_MODE =
     buildConf("spark.gluten.velox.fs.s3a.retry.mode")
       .internal()
-      .doc("Retry mode for AWS s3 connection error.")
+      .doc("Retry mode for AWS s3 connection error: legacy, standard and adaptive.")
       .stringConf
       .createWithDefault("legacy")
 
@@ -1960,7 +1960,7 @@ object GlutenConfig {
       .internal()
       .doc("Timeout for AWS s3 connection.")
       .stringConf
-      .createWithDefault("1s")
+      .createWithDefault("200s")
 
   val VELOX_ORC_SCAN_ENABLED =
     buildStaticConf("spark.gluten.sql.columnar.backend.velox.orc.scan.enabled")