#625: Make Dataproc init script timeout configurable (#628)

* add defaultExecutionTimeout from config

* set the timeout to 30 mins in reference.conf, can override in leo.conf

src/main/resources/reference.conf

@@ -3,6 +3,7 @@
 dataproc {
   applicationName = "firecloud:leonardo"
   dataprocDockerImage = "gcr.io/broad-dsde-prod/leonardo-notebooks:prod"
+  defaultExecutionTimeout = 30 minutes
   jupyterServerName = "jupyter-server"
   firewallRuleName = "leonardo-notebooks-rule"
   networkTag = "leonardo"

src/main/scala/org/broadinstitute/dsde/workbench/leonardo/Boot.scala

@@ -77,7 +77,7 @@ object Boot extends App with LazyLogging {
     }
 
     val (leoServiceAccountEmail, leoServiceAccountPemFile) = serviceAccountProvider.getLeoServiceAccountAndKey
-    val gdDAO = new HttpGoogleDataprocDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google", NetworkTag(dataprocConfig.networkTag), dataprocConfig.vpcNetwork.map(VPCNetworkName), dataprocConfig.vpcSubnet.map(VPCSubnetName), dataprocConfig.dataprocDefaultRegion)
+    val gdDAO = new HttpGoogleDataprocDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google", NetworkTag(dataprocConfig.networkTag), dataprocConfig.vpcNetwork.map(VPCNetworkName), dataprocConfig.vpcSubnet.map(VPCSubnetName), dataprocConfig.dataprocDefaultRegion, dataprocConfig.defaultExecutionTimeout)
     val googleComputeDAO = new HttpGoogleComputeDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google")
     val googleIamDAO = new HttpGoogleIamDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google")
     val googleStorageDAO = new HttpGoogleStorageDAO(dataprocConfig.applicationName, Pem(leoServiceAccountEmail, leoServiceAccountPemFile), "google")

src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/DataprocConfig.scala

@@ -2,12 +2,15 @@ package org.broadinstitute.dsde.workbench.leonardo.config
 
 import org.broadinstitute.dsde.workbench.model.google.GoogleProject
 
+import scala.concurrent.duration.FiniteDuration
+
 case class DataprocConfig(
                            applicationName: String,
                            dataprocDefaultRegion: String,
                            leoGoogleProject: GoogleProject,
                            dataprocDockerImage: String,
                            clusterUrlBase: String,
+                           defaultExecutionTimeout: FiniteDuration,
                            jupyterServerName: String,
                            firewallRuleName: String,
                            networkTag: String,

src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/package.scala

@@ -23,6 +23,7 @@ package object config {
       GoogleProject(config.getString("leoGoogleProject")),
       config.getString("dataprocDockerImage"),
       config.getString("clusterUrlBase"),
+      toScalaDuration(config.getDuration("defaultExecutionTimeout")),
       config.getString("jupyterServerName"),
       config.getString("firewallRuleName"),
       config.getString("networkTag"),

src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/google/HttpGoogleDataprocDAO.scala

@@ -27,6 +27,7 @@ import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GcsPath, G
 import org.broadinstitute.dsde.workbench.model.{UserInfo, WorkbenchEmail, WorkbenchException, WorkbenchUserId}
 
 import scala.collection.JavaConverters._
+import scala.concurrent.duration.FiniteDuration
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Try
 
@@ -36,7 +37,8 @@ class HttpGoogleDataprocDAO(appName: String,
                             networkTag: NetworkTag,
                             vpcNetwork: Option[VPCNetworkName],
                             vpcSubnet: Option[VPCSubnetName],
-                            defaultRegion: String)
+                            defaultRegion: String,
+                            defaultExecutionTimeout: FiniteDuration)
                            (implicit override val system: ActorSystem, override val executionContext: ExecutionContext)
   extends AbstractHttpGoogleDAO(appName, googleCredentialMode, workbenchMetricBaseName) with GoogleDataprocDAO {
 
@@ -238,7 +240,7 @@ class HttpGoogleDataprocDAO(appName: String,
 
     // Create a NodeInitializationAction, which specifies the executable to run on a node.
     // This executable is our init-actions.sh, which will stand up our jupyter server and proxy.
-    val initActions = Seq(new NodeInitializationAction().setExecutableFile(initScript.toUri))
+    val initActions = Seq(new NodeInitializationAction().setExecutableFile(initScript.toUri).setExecutionTimeout(finiteDurationToGoogleDuration(defaultExecutionTimeout)))
 
     // Create a config for the master node, if properties are not specified in request, use defaults
     val masterConfig = new InstanceGroupConfig()
@@ -411,6 +413,11 @@ class HttpGoogleDataprocDAO(appName: String,
     } yield parseZone(zoneUri)
   }
 
+  //Note that this conversion will shave off anything smaller than a second in magnitude
+  private def finiteDurationToGoogleDuration(duration: FiniteDuration): String = {
+    s"${duration.toSeconds}s"
+  }
+
   private implicit class GoogleExceptionSupport[A](future: Future[A]) {
     def handleGoogleException(project: GoogleProject, context: Option[String] = None): Future[A] = {
       future.recover {

src/test/resources/reference.conf

@@ -35,6 +35,7 @@ dataproc {
   leoGoogleProject = "test-bucket"
   dataprocDockerImage = "testrepo/test"
   clusterUrlBase = "http://leonardo/"
+  defaultExecutionTimeout = 30 minutes
   jupyterServerName = "test-server"
   createClusterAsPetServiceAccount = false
   firewallRuleName = "test-rule"