Skip to content
This repository has been archived by the owner on Apr 24, 2020. It is now read-only.

Commit

Permalink
parse and pass spark.executor.extraJavaOptions to executors
Browse files Browse the repository at this point in the history
  • Loading branch information
cgbaker authored and lgfa29 committed Oct 29, 2019
1 parent 7ad9b8d commit 8309661
Showing 1 changed file with 11 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import com.hashicorp.nomad.apimodel.Task

import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.internal.config.EXECUTOR_MEMORY
import org.apache.spark.util.Utils

private[spark] object ExecutorTask
extends SparkNomadTaskType("executor", "executor", EXECUTOR_MEMORY) {
Expand Down Expand Up @@ -48,8 +49,7 @@ private[spark] object ExecutorTask

conf.getExecutorEnv.foreach((task.addEnv _).tupled)

val executorConf: Seq[(String, String)] = {

val executorConf = {
val explicitConf = Seq(
"spark.executor.port" -> executorPort.placeholder,
"spark.blockManager.port" -> blockManagerPort.placeholder
Expand All @@ -61,14 +61,17 @@ private[spark] object ExecutorTask
) && !PROPERTIES_NOT_TO_FORWARD_TO_EXECUTOR.contains(name)
}

explicitConf ++ forwardedConf
(explicitConf ++ forwardedConf.toSeq).map {
case (k, v) => s"-D$k=$v"
.replaceAllLiterally("\\", "\\\\")
.replaceAllLiterally("\"", "\\\"")
}.map('"' + _ + '"')
}

task.addEnv("SPARK_EXECUTOR_OPTS",
executorConf.map { case (k, v) => s"-D$k=$v"
.replaceAllLiterally("\\", "\\\\")
.replaceAllLiterally("\"", "\\\"")
}.map('"' + _ + '"').mkString(" "))
val extraJavaOpts = conf.getOption("spark.executor.extraJavaOptions")
.map(Utils.splitCommandString).getOrElse(Seq.empty)

task.addEnv("SPARK_EXECUTOR_OPTS", (extraJavaOpts ++ executorConf).mkString(" "))

task.addEnv("SPARK_EXECUTOR_MEMORY", jvmMemory(conf, task))

Expand Down

0 comments on commit 8309661

Please sign in to comment.