better handling of local:// protocol URIs

hashicorp · Oct 29, 2019 · 7ad9b8d · 7ad9b8d
1 parent b349efa
commit 7ad9b8d
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/...e-managers/nomad/src/main/scala/org/apache/spark/scheduler/cluster/nomad/DriverTask.scala b/...e-managers/nomad/src/main/scala/org/apache/spark/scheduler/cluster/nomad/DriverTask.scala
@@ -78,7 +78,7 @@ private[spark] object DriverTask extends SparkNomadTaskType("driver", "driver",
       .map(_.split(",").filter(_.nonEmpty))
       .filter(_.nonEmpty)
       .foreach { files =>
-        conf.set("spark.files", files.map(asFileIn(jobConf, task)).map(f => f.stripPrefix("local://")).mkString(","))
+        conf.set("spark.files", files.map(asFileIn(jobConf, task)).mkString(","))
       }
 
     val driverConf: Seq[(String, String)] = {

diff --git a/...rs/nomad/src/main/scala/org/apache/spark/scheduler/cluster/nomad/SparkNomadTaskType.scala b/...rs/nomad/src/main/scala/org/apache/spark/scheduler/cluster/nomad/SparkNomadTaskType.scala
@@ -191,7 +191,10 @@ private[spark] abstract class SparkNomadTaskType(
   protected def asFileIn(jobConf: SparkNomadJob.CommonConf, task: Task)(url: String): String = {
     val (file, artifact) = asFileAndArtifact(jobConf, new URI(url), false)
     artifact.foreach(task.addArtifacts(_))
-    file.toString
+    file.getScheme match {
+      case "local" => file.getPath
+      case _ => file.toString
+    }
   }
 
   protected def jvmMemory(conf: SparkConf, task: Task): String = {