Auto deploy feed version

pom.xml

@@ -270,7 +270,7 @@
         <dependency>
             <groupId>com.github.conveyal</groupId>
             <artifactId>gtfs-lib</artifactId>
-            <version>6.2.2</version>
+            <version>6.2.4</version>
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
             -->

src/main/java/com/conveyal/datatools/common/status/FeedSourceJob.java

@@ -0,0 +1,14 @@
+package com.conveyal.datatools.common.status;
+
+import com.conveyal.datatools.manager.auth.Auth0UserProfile;
+
+/**
+ * This class should be used for any job that operates on a FeedSource.
+ */
+public abstract class FeedSourceJob extends MonitorableJob {
+    public FeedSourceJob(Auth0UserProfile owner, String name, JobType type) {
+        super(owner, name, type);
+    }
+
+    public abstract String getFeedSourceId();
+}

src/main/java/com/conveyal/datatools/common/status/FeedVersionJob.java

@@ -0,0 +1,14 @@
+package com.conveyal.datatools.common.status;
+
+import com.conveyal.datatools.manager.auth.Auth0UserProfile;
+
+/**
+ * This class should be used for any job that operates on a FeedVersion.
+ */
+public abstract class FeedVersionJob extends FeedSourceJob {
+    public FeedVersionJob(Auth0UserProfile owner, String name, JobType type) {
+        super(owner, name, type);
+    }
+
+    public abstract String getFeedVersionId();
+}

src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java

@@ -1,7 +1,7 @@
 package com.conveyal.datatools.common.status;
 
-import com.conveyal.datatools.manager.DataManager;
 import com.conveyal.datatools.manager.auth.Auth0UserProfile;
+import com.conveyal.datatools.manager.utils.JobUtils;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -18,8 +18,6 @@
 import java.util.Set;
 import java.util.UUID;
 
-import static com.conveyal.datatools.manager.controllers.api.StatusController.getJobsForUser;
-
 /**
  * Created by landon on 6/13/16.
  */
@@ -31,8 +29,15 @@ public abstract class MonitorableJob implements Runnable, Serializable {
     // Public fields will be serialized over HTTP API and visible to the web client
     public final JobType type;
     public File file;
-    public String parentJobId;
-    public JobType parentJobType;
+
+    /**
+     * Whether the job is currently running. This is needed since some jobs can be recurring jobs that won't run until
+     * their scheduled time and when they finish they could run again.
+     */
+    public boolean active = false;
+
+    protected String parentJobId;
+    protected JobType parentJobType;
     // Status is not final to allow some jobs to have extra status fields.
     public Status status = new Status();
     // Name is not final in case it needs to be amended during job processing.
@@ -48,6 +53,7 @@ public abstract class MonitorableJob implements Runnable, Serializable {
     public List<MonitorableJob> subJobs = new ArrayList<>();
 
     public enum JobType {
+        AUTO_DEPLOY_FEED_VERSION,
         UNKNOWN_TYPE,
         ARBITRARY_FEED_TRANSFORM,
         BUILD_TRANSPORT_NETWORK,
@@ -102,9 +108,9 @@ public MonitorableJob () {
     private void registerJob() {
         // Get all active jobs and add the latest active job. Note: Removal of job from user's set of jobs is handled
         // in the StatusController when a user requests their active jobs and the job has finished/errored.
-        Set<MonitorableJob> userJobs = getJobsForUser(this.owner);
+        Set<MonitorableJob> userJobs = JobUtils.getJobsForUser(this.owner);
         userJobs.add(this);
-        DataManager.userJobsMap.put(retrieveUserId(), userJobs);
+        JobUtils.userJobsMap.put(retrieveUserId(), userJobs);
     }
 
     @JsonProperty("owner")
@@ -117,6 +123,11 @@ public String retrieveEmail() {
         return this.owner.getEmail();
     }
 
+    @JsonIgnore @BsonIgnore
+    public List<MonitorableJob> getSubJobs() {
+        return subJobs;
+    }
+
     public File retrieveFile () {
         return file;
     }
@@ -140,6 +151,7 @@ public void jobFinished () {
      * override jobLogic and jobFinished method(s).
      */
     public void run () {
+        active = true;
         boolean parentJobErrored = false;
         boolean subTaskErrored = false;
         String cancelMessage = "";
@@ -195,8 +207,10 @@ public void run () {
             // could be displayed by the client.
         } catch (Exception e) {
             status.fail("Job failed due to unhandled exception!", e);
+        } finally {
+            LOG.info("{} (jobId={}) {} in {} ms", type, jobId, status.error ? "errored" : "completed", status.duration);
+            active = false;
         }
-        LOG.info("{} (jobId={}) {} in {} ms", type, jobId, status.error ? "errored" : "completed", status.duration);
     }
 
     /**

src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java

@@ -16,6 +16,7 @@
 import com.conveyal.datatools.manager.models.JsonViews;
 import com.conveyal.datatools.manager.models.Snapshot;
 import com.conveyal.datatools.manager.persistence.Persistence;
+import com.conveyal.datatools.manager.utils.JobUtils;
 import com.conveyal.datatools.manager.utils.json.JsonManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -101,7 +102,7 @@ private static String createSnapshot (Request req, Response res) throws IOExcept
             createSnapshotJob.addNextJob(new CreateFeedVersionFromSnapshotJob(feedSource, snapshot, userProfile));
         }
         // Begin asynchronous execution.
-        DataManager.heavyExecutor.execute(createSnapshotJob);
+        JobUtils.heavyExecutor.execute(createSnapshotJob);
         return SparkUtils.formatJobMessage(createSnapshotJob.jobId, "Creating snapshot.");
     }
 
@@ -122,7 +123,7 @@ private static String importFeedVersionAsSnapshot(Request req, Response res) {
         boolean preserveBuffer = "true".equals(req.queryParams("preserveBuffer")) && feedSource.editorNamespace != null;
         CreateSnapshotJob createSnapshotJob =
                 new CreateSnapshotJob(userProfile, snapshot, true, false, preserveBuffer);
-        DataManager.heavyExecutor.execute(createSnapshotJob);
+        JobUtils.heavyExecutor.execute(createSnapshotJob);
         return formatJobMessage(createSnapshotJob.jobId, "Importing version as snapshot.");
     }
 
@@ -161,7 +162,7 @@ private static String restoreSnapshot (Request req, Response res) {
         String name = "Restore snapshot " + snapshotToRestore.name;
         Snapshot snapshot = new Snapshot(name, feedSource.id, snapshotToRestore.namespace);
         CreateSnapshotJob createSnapshotJob = new CreateSnapshotJob(userProfile, snapshot, true, false, preserveBuffer);
-        DataManager.heavyExecutor.execute(createSnapshotJob);
+        JobUtils.heavyExecutor.execute(createSnapshotJob);
         return formatJobMessage(createSnapshotJob.jobId, "Restoring snapshot...");
     }
 
@@ -175,7 +176,7 @@ private static String downloadSnapshotAsGTFS(Request req, Response res) {
         // Create and kick off export job.
         // FIXME: what if a snapshot is already written to S3?
         ExportSnapshotToGTFSJob exportSnapshotToGTFSJob = new ExportSnapshotToGTFSJob(userProfile,  snapshot);
-        DataManager.heavyExecutor.execute(exportSnapshotToGTFSJob);
+        JobUtils.heavyExecutor.execute(exportSnapshotToGTFSJob);
         return formatJobMessage(exportSnapshotToGTFSJob.jobId, "Exporting snapshot to GTFS.");
     }
 

src/main/java/com/conveyal/datatools/editor/datastore/FeedTx.java

@@ -1,6 +1,19 @@
 package com.conveyal.datatools.editor.datastore;
 
-import com.conveyal.datatools.editor.models.transit.*;
+
+import com.conveyal.datatools.editor.models.transit.Agency;
+import com.conveyal.datatools.editor.models.transit.AttributeAvailabilityType;
+import com.conveyal.datatools.editor.models.transit.EditorFeed;
+import com.conveyal.datatools.editor.models.transit.Fare;
+import com.conveyal.datatools.editor.models.transit.Route;
+import com.conveyal.datatools.editor.models.transit.ScheduleException;
+import com.conveyal.datatools.editor.models.transit.ServiceCalendar;
+import com.conveyal.datatools.editor.models.transit.StatusType;
+import com.conveyal.datatools.editor.models.transit.Stop;
+import com.conveyal.datatools.editor.models.transit.StopTime;
+import com.conveyal.datatools.editor.models.transit.Trip;
+import com.conveyal.datatools.editor.models.transit.TripPattern;
+import com.conveyal.datatools.editor.models.transit.TripPatternStop;
 import com.conveyal.datatools.editor.utils.GeoUtils;
 import com.conveyal.gtfs.GTFSFeed;
 import com.conveyal.gtfs.model.CalendarDate;

src/main/java/com/conveyal/datatools/manager/ConvertMain.java

@@ -6,9 +6,9 @@
 import com.conveyal.datatools.editor.jobs.ConvertEditorMapDBToSQL;
 import com.conveyal.datatools.editor.models.Snapshot;
 import com.conveyal.datatools.manager.controllers.DumpController;
-import com.conveyal.datatools.manager.controllers.api.StatusController;
 import com.conveyal.datatools.manager.models.FeedSource;
 import com.conveyal.datatools.manager.persistence.Persistence;
+import com.conveyal.datatools.manager.utils.JobUtils;
 import org.apache.commons.io.FileUtils;
 import org.mapdb.Fun;
 import org.slf4j.Logger;
@@ -97,15 +97,15 @@ public static void main(String[] args) throws Exception {
         // STEP 3A: For each snapshot/editor DB, create a snapshot Mongo object for the feed source with the FeedLoadResult.
         migrateEditorFeeds();
         LOG.info("Done queueing!!!!!!!!");
-        int totalJobs = StatusController.getAllJobs().size();
-        while (!StatusController.filterActiveJobs(StatusController.getAllJobs()).isEmpty()) {
+        int totalJobs = JobUtils.getAllJobs().size();
+        while (!JobUtils.filterActiveJobs(JobUtils.getAllJobs()).isEmpty()) {
             // While there are still active jobs, continue waiting.
-            Set<MonitorableJob> activeJobs = StatusController.filterActiveJobs(StatusController.getAllJobs());
+            Set<MonitorableJob> activeJobs = JobUtils.filterActiveJobs(JobUtils.getAllJobs());
             LOG.info(String.format("%d/%d jobs still active. Checking for completion again in 5 seconds...", activeJobs.size(), totalJobs));
 //            LOG.info(String.join(", ", activeJobs.stream().map(job -> job.name).collect(Collectors.toList())));
-            int jobsInExecutor = ((ThreadPoolExecutor) DataManager.heavyExecutor).getActiveCount();
+            int jobsInExecutor = ((ThreadPoolExecutor) JobUtils.heavyExecutor).getActiveCount();
             LOG.info(String.format("Jobs in thread pool executor: %d", jobsInExecutor));
-            LOG.info(String.format("Jobs completed by executor: %d", ((ThreadPoolExecutor) DataManager.heavyExecutor).getCompletedTaskCount()));
+            LOG.info(String.format("Jobs completed by executor: %d", ((ThreadPoolExecutor) JobUtils.heavyExecutor).getCompletedTaskCount()));
             Thread.sleep(5000);
         }
         long durationInMillis = System.currentTimeMillis() - startTime;
@@ -141,11 +141,11 @@ public static boolean migrateEditorFeeds (String ...feedIdsToSkip) {
                     if (!feedSourcesEncountered.contains(feedSource.id)) {
                         // If this is the first feed encountered, load the editor buffer.
                         ConvertEditorMapDBToSQL convertEditorBufferToSQL = new ConvertEditorMapDBToSQL(snapshot.id.a, null);
-                        DataManager.heavyExecutor.execute(convertEditorBufferToSQL);
+                        JobUtils.heavyExecutor.execute(convertEditorBufferToSQL);
                         count++;
                     }
                     ConvertEditorMapDBToSQL convertEditorMapDBToSQL = new ConvertEditorMapDBToSQL(snapshot.id.a, snapshot.id.b);
-                    DataManager.heavyExecutor.execute(convertEditorMapDBToSQL);
+                    JobUtils.heavyExecutor.execute(convertEditorMapDBToSQL);
                     LOG.info(count + "/" + snapshotCount + " snapshot conversion queued");
                     feedSourcesEncountered.add(feedSource.id);
                     count++;

src/main/java/com/conveyal/datatools/manager/DataManager.java

@@ -87,21 +87,9 @@ public class DataManager {
     // TODO: define type for ExternalFeedResource Strings
     public static final Map<String, ExternalFeedResource> feedResources = new HashMap<>();
 
-    /**
-     * Stores jobs underway by user ID. NOTE: any set created and stored here must be created with
-     * {@link Sets#newConcurrentHashSet()} or similar thread-safe Set.
-     */
-    public static Map<String, Set<MonitorableJob>> userJobsMap = new ConcurrentHashMap<>();
-
     // ObjectMapper that loads in YAML config files
     private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
 
-
-    // Heavy executor should contain long-lived CPU-intensive tasks (e.g., feed loading/validation)
-    public static Executor heavyExecutor = Executors.newFixedThreadPool(4);
-    // light executor is for tasks for things that should finish quickly (e.g., email notifications)
-    public static Executor lightExecutor = Executors.newSingleThreadExecutor();
-
     public static String repoUrl;
     public static String commit = "";
 

src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java

@@ -1,7 +1,6 @@
 package com.conveyal.datatools.manager.controllers;
 
 import com.conveyal.datatools.common.status.MonitorableJob;
-import com.conveyal.datatools.manager.DataManager;
 import com.conveyal.datatools.manager.auth.Auth0UserProfile;
 import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob;
 import com.conveyal.datatools.manager.jobs.ValidateFeedJob;
@@ -15,6 +14,7 @@
 import com.conveyal.datatools.manager.models.Project;
 import com.conveyal.datatools.manager.models.Snapshot;
 import com.conveyal.datatools.manager.persistence.Persistence;
+import com.conveyal.datatools.manager.utils.JobUtils;
 import com.conveyal.datatools.manager.utils.json.JsonManager;
 import com.conveyal.gtfs.validator.ValidationResult;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -365,7 +365,7 @@ public static boolean validateAll (boolean load, boolean force, String filterFee
             } else {
                 job = new ValidateFeedJob(version, systemUser, false);
             }
-            DataManager.heavyExecutor.execute(job);
+            JobUtils.heavyExecutor.execute(job);
         }
         // ValidateAllFeedsJob validateAllFeedsJob = new ValidateAllFeedsJob("system", force, load);
         return true;