Auto deploy feed version

src/main/java/com/conveyal/datatools/common/status/FeedSourceJob.java

@@ -0,0 +1,14 @@
+package com.conveyal.datatools.common.status;
+
+import com.conveyal.datatools.manager.auth.Auth0UserProfile;
+
+/**
+ * This class should be used for any job that operates on a FeedSource.
+ */
+public abstract class FeedSourceJob extends MonitorableJob {
+    public FeedSourceJob(Auth0UserProfile owner, String name, JobType type) {
+        super(owner, name, type);
+    }
+
+    public abstract String getFeedSourceId();
+}

src/main/java/com/conveyal/datatools/common/status/FeedVersionJob.java

@@ -0,0 +1,14 @@
+package com.conveyal.datatools.common.status;
+
+import com.conveyal.datatools.manager.auth.Auth0UserProfile;
+
+/**
+ * This class should be used for any job that operates on a FeedVersion.
+ */
+public abstract class FeedVersionJob extends FeedSourceJob {
+    public FeedVersionJob(Auth0UserProfile owner, String name, JobType type) {
+        super(owner, name, type);
+    }
+
+    public abstract String getFeedVersionId();
+}

src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java

@@ -2,8 +2,10 @@
 
 import com.conveyal.datatools.manager.DataManager;
 import com.conveyal.datatools.manager.auth.Auth0UserProfile;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.bson.codecs.pojo.annotations.BsonIgnore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,8 +31,8 @@ public abstract class MonitorableJob implements Runnable, Serializable {
     // Public fields will be serialized over HTTP API and visible to the web client
     public final JobType type;
     public File file;
-    public String parentJobId;
-    public JobType parentJobType;
+    protected String parentJobId;
+    protected JobType parentJobType;
     // Status is not final to allow some jobs to have extra status fields.
     public Status status = new Status();
     // Name is not final in case it needs to be amended during job processing.
@@ -41,9 +43,10 @@ public abstract class MonitorableJob implements Runnable, Serializable {
      * Additional jobs that will be run after the main logic of this job has completed.
      * This job is not considered entirely completed until its sub-jobs have all completed.
      */
-    protected List<MonitorableJob> subJobs = new ArrayList<>();
+    final private List<MonitorableJob> subJobs = new ArrayList<>();
 
     public enum JobType {
+        AUTO_DEPLOY_FEED_VERSION,
         UNKNOWN_TYPE,
         ARBITRARY_FEED_TRANSFORM,
         BUILD_TRANSPORT_NETWORK,
@@ -113,6 +116,11 @@ public String retrieveEmail() {
         return this.owner.getEmail();
     }
 
+    @JsonIgnore @BsonIgnore
+    public List<MonitorableJob> getSubJobs() {
+        return subJobs;
+    }
+
     public File retrieveFile () {
         return file;
     }
@@ -288,15 +296,16 @@ private String getCallingMethodTrace() {
          * Shorthand method to update status object on successful job completion.
          */
         public void completeSuccessfully(String message) {
-            this.complete(false, message);
+            // Do not overwrite the message (and other fields), if the job has already been completed.
+            if (!this.completed) this.complete(false, message);
         }
 
         /**
          * Set job status to completed with error and message information.
          */
         private void complete(boolean isError, String message) {
             this.error = isError;
-            // Skip message update if null.
+            // Skip message update if the job message is null or the message has already been defined.
             if (message != null) this.message = message;
             this.percentComplete = 100;
             this.completed = true;

src/main/java/com/conveyal/datatools/editor/datastore/FeedTx.java

@@ -1,6 +1,19 @@
 package com.conveyal.datatools.editor.datastore;
 
-import com.conveyal.datatools.editor.models.transit.*;
+
+import com.conveyal.datatools.editor.models.transit.Agency;
+import com.conveyal.datatools.editor.models.transit.AttributeAvailabilityType;
+import com.conveyal.datatools.editor.models.transit.EditorFeed;
+import com.conveyal.datatools.editor.models.transit.Fare;
+import com.conveyal.datatools.editor.models.transit.Route;
+import com.conveyal.datatools.editor.models.transit.ScheduleException;
+import com.conveyal.datatools.editor.models.transit.ServiceCalendar;
+import com.conveyal.datatools.editor.models.transit.StatusType;
+import com.conveyal.datatools.editor.models.transit.Stop;
+import com.conveyal.datatools.editor.models.transit.StopTime;
+import com.conveyal.datatools.editor.models.transit.Trip;
+import com.conveyal.datatools.editor.models.transit.TripPattern;
+import com.conveyal.datatools.editor.models.transit.TripPatternStop;
 import com.conveyal.datatools.editor.utils.GeoUtils;
 import com.conveyal.gtfs.GTFSFeed;
 import com.conveyal.gtfs.model.CalendarDate;

src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java

@@ -18,6 +18,7 @@
 import com.conveyal.datatools.manager.models.Project;
 import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.datatools.manager.utils.json.JsonManager;
+import com.mongodb.client.FindIterable;
 import org.bson.Document;
 import org.eclipse.jetty.http.HttpStatus;
 import org.slf4j.Logger;
@@ -49,7 +50,7 @@
  */
 public class DeploymentController {
     private static final Logger LOG = LoggerFactory.getLogger(DeploymentController.class);
-    private static Map<String, DeployJob> deploymentJobsByServer = new HashMap<>();
+    private static final Map<String, DeployJob> deploymentJobsByServer = new HashMap<>();
 
     /**
      * Gets the deployment specified by the request's id parameter and ensure that user has access to the
@@ -376,6 +377,42 @@ private static boolean terminateEC2InstanceForDeployment(Request req, Response r
         return true;
     }
 
+    /**
+     * Queue a new {@link DeployJob} if there are no conflicting jobs assigned to the specified server.
+     * @param deployJob new deploy job to queue
+     * @return whether the deploy job was successfully queued or not
+     */
+    public static boolean queueDeployJob(DeployJob deployJob) {
+        String serverId = deployJob.getServerId();
+        // Check that we can deploy to the specified target. (Any deploy job for the target that is presently active will
+        // cause a halt.)
+        if (deploymentJobsByServer.containsKey(serverId)) {
+            // There is a deploy job for the server. Check if it is active.
+            DeployJob conflictingDeployJob = deploymentJobsByServer.get(serverId);
+            if (conflictingDeployJob != null && !conflictingDeployJob.status.completed) {
+                // Another deploy job is actively being deployed to the server target.
+                LOG.error("New deploy job will not be queued due to active deploy job in progress.");
+                return false;
+            }
+        }
+
+        // For any previous deployments sent to the server/router combination, set deployedTo to null because
+        // this new one will overwrite it. NOTE: deployedTo for the current deployment will only be updated after the
+        // successful completion of the deploy job.
+        FindIterable<Deployment> deploymentsWithSameTarget = Deployment.retrieveDeploymentForServerAndRouterId(
+            serverId,
+            deployJob.getDeployment().routerId
+        );
+        for (Deployment oldDeployment : deploymentsWithSameTarget) {
+            LOG.info("Setting deployment target to null for id={}", oldDeployment.id);
+            Persistence.deployments.updateField(oldDeployment.id, "deployedTo", null);
+        }
+        // Finally, add deploy job to the heavy executor.
+        DataManager.heavyExecutor.execute(deployJob);
+        deploymentJobsByServer.put(serverId, deployJob);
+        return true;
+    }
+
     /**
      * HTTP controller to fetch information about provided EC2 machines that power ELBs running a trip planner.
      */
@@ -396,8 +433,9 @@ private static String deploy (Request req, Response res) {
         String target = req.params("target");
         Deployment deployment = getDeploymentWithPermissions(req, res);
         Project project = Persistence.projects.getById(deployment.projectId);
-        if (project == null)
+        if (project == null) {
             logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid");
+        }
         // Get server by ID
         OtpServer otpServer = Persistence.servers.getById(target);
         if (otpServer == null) {
@@ -411,23 +449,6 @@ private static String deploy (Request req, Response res) {
             logMessageAndHalt(req, 401, "User not authorized to deploy to admin-only target OTP server.");
         }
 
-        // Check that we can deploy to the specified target. (Any deploy job for the target that is presently active will
-        // cause a halt.)
-        if (deploymentJobsByServer.containsKey(target)) {
-            // There is a deploy job for the server. Check if it is active.
-            DeployJob deployJob = deploymentJobsByServer.get(target);
-            if (deployJob != null && !deployJob.status.completed) {
-                // Job for the target is still active! Send a 202 to the requester to indicate that it is not possible
-                // to deploy to this target right now because someone else is deploying.
-                String message = String.format(
-                        "Will not process request to deploy %s. Deployment currently in progress for target: %s",
-                        deployment.name,
-                        target);
-                LOG.warn(message);
-                logMessageAndHalt(req, HttpStatus.ACCEPTED_202, message);
-            }
-        }
-
         // Get the URLs to deploy to.
         List<String> targetUrls = otpServer.internalUrl;
         if ((targetUrls == null || targetUrls.isEmpty()) && (otpServer.s3Bucket == null || otpServer.s3Bucket.isEmpty())) {
@@ -438,19 +459,17 @@ private static String deploy (Request req, Response res) {
             );
         }
 
-        // For any previous deployments sent to the server/router combination, set deployedTo to null because
-        // this new one will overwrite it. NOTE: deployedTo for the current deployment will only be updated after the
-        // successful completion of the deploy job.
-        for (Deployment oldDeployment : Deployment.retrieveDeploymentForServerAndRouterId(target, deployment.routerId)) {
-            LOG.info("Setting deployment target to null id={}", oldDeployment.id);
-            Persistence.deployments.updateField(oldDeployment.id, "deployedTo", null);
-        }
-
         // Execute the deployment job and keep track of it in the jobs for server map.
         DeployJob job = new DeployJob(deployment, userProfile, otpServer);
-        DataManager.heavyExecutor.execute(job);
-        deploymentJobsByServer.put(target, job);
-
+        if (!queueDeployJob(job)) {
+            // Job for the target is still active! Send a 202 to the requester to indicate that it is not possible
+            // to deploy to this target right now because someone else is deploying.
+            String message = String.format(
+                "Will not process request to deploy %s. Deployment currently in progress for target: %s",
+                deployment.name,
+                target);
+            logMessageAndHalt(req, HttpStatus.ACCEPTED_202, message);
+        }
         return SparkUtils.formatJobMessage(job.jobId, "Deployment initiating.");
     }
 

src/main/java/com/conveyal/datatools/manager/controllers/api/StatusController.java

@@ -12,16 +12,13 @@
 import spark.Request;
 import spark.Response;
 
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
-import static spark.Spark.delete;
 import static spark.Spark.get;
 
 /**