Skip to content
This repository has been archived by the owner on Apr 24, 2020. It is now read-only.

Commit

Permalink
Disable go-getter auto-unzip
Browse files Browse the repository at this point in the history
Added a parameter to set `archive=false` option on all of the files to download except for the spark tarball which needs to be unpacked as part of the job deployment.
  • Loading branch information
angrycub authored and barnardb committed Jun 14, 2018
1 parent 1d4d32e commit 3eef702
Showing 1 changed file with 8 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ private[spark] abstract class SparkNomadTaskType(
s"You must either set ${SPARK_DISTRIBUTION.key} or provide a ${JOB_TEMPLATE.key} " +
s"""with a command provided for the task with meta spark.nomad.role = "$role"""")
}
val (sparkHomeUrl, sparkArtifact) = asFileAndArtifact(jobConf, sparkDistributionUrl)
val (sparkHomeUrl, sparkArtifact) = asFileAndArtifact(jobConf, sparkDistributionUrl, true)
sparkArtifact.foreach(task.addArtifacts(_))
val sparkDir =
if (sparkDistributionUrl.getScheme == "local") sparkHomeUrl.getPath
Expand All @@ -185,7 +185,7 @@ private[spark] abstract class SparkNomadTaskType(
}

protected def asFileIn(jobConf: SparkNomadJob.CommonConf, task: Task)(url: String): String = {
val (file, artifact) = asFileAndArtifact(jobConf, new URI(url))
val (file, artifact) = asFileAndArtifact(jobConf, new URI(url), false)
artifact.foreach(task.addArtifacts(_))
file.toString
}
Expand Down Expand Up @@ -266,7 +266,8 @@ private[spark] abstract class SparkNomadTaskType(

private def asFileAndArtifact(
jobConf: SparkNomadJob.CommonConf,
url: URI
url: URI,
unarchive: Boolean
): (URI, Option[TaskArtifact]) = {
url.getScheme match {
case "local" => url -> None
Expand All @@ -285,6 +286,10 @@ private[spark] abstract class SparkNomadTaskType(
val artifact = new TaskArtifact()
.setRelativeDest(workDir)
.setGetterSource(url.toString)
if (!unarchive) {
val options = Map("archive" -> false)
artifact.setGetterOptions(options.asJava)
}
file -> Some(artifact)
}
}
Expand Down

0 comments on commit 3eef702

Please sign in to comment.