From 1e369f2a5d480223343b87d21290bce591678eaa Mon Sep 17 00:00:00 2001 From: ulysses-you Date: Fri, 21 Jun 2024 16:06:52 +0800 Subject: [PATCH] Use sc.listFiles instead addedFiles.keys --- .../src/main/scala/org/apache/spark/HdfsConfGenerator.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala index 9756837d96e5..04272517e5bb 100644 --- a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala +++ b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala @@ -41,8 +41,8 @@ object HdfsConfGenerator extends Logging { addFileMethod.invoke(sc, path, Boolean.box(false), Boolean.box(true), Boolean.box(false)) // Overwrite the spark internal config `spark.app.initial.file.urls`, // so that the file can be available before initializing executor plugin. - assert(sc.addedFiles.nonEmpty) - sc.conf.set("spark.app.initial.file.urls", sc.addedFiles.keys.toSeq.mkString(",")) + assert(sc.listFiles.nonEmpty) + sc.conf.set("spark.app.initial.file.urls", sc.listFiles().mkString(",")) } private def ignoreKey(key: String): Boolean = {