Deploy OTP to AWS load balancer and manage OTP servers in separate collection (and misc other fixes)

src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java

@@ -7,6 +7,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.Serializable;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -18,7 +19,8 @@
 /**
  * Created by landon on 6/13/16.
  */
-public abstract class MonitorableJob implements Runnable {
+public abstract class MonitorableJob implements Runnable, Serializable {
+    private static final long serialVersionUID = 1L;
     private static final Logger LOG = LoggerFactory.getLogger(MonitorableJob.class);
     public final String owner;
 
@@ -129,7 +131,6 @@ public void run () {
         boolean parentJobErrored = false;
         boolean subTaskErrored = false;
         String cancelMessage = "";
-        long startTimeNanos = System.nanoTime();
         try {
             // First execute the core logic of the specific MonitorableJob subclass
             jobLogic();
@@ -188,8 +189,7 @@ public void run () {
             LOG.error("Job failed", ex);
             status.update(true, ex.getMessage(), 100, true);
         }
-        status.startTime = TimeUnit.NANOSECONDS.toMillis(startTimeNanos);
-        status.duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos);
+        status.duration = System.currentTimeMillis() - status.startTime;
         LOG.info("{} {} {} in {} ms", type, jobId, status.error ? "errored" : "completed", status.duration);
     }
 
@@ -243,7 +243,7 @@ public static class Status {
         /** How much of task is complete? */
         public double percentComplete;
 
-        public long startTime;
+        public long startTime = System.currentTimeMillis();
         public long duration;
 
         // When was the job initialized?

src/main/java/com/conveyal/datatools/manager/auth/Auth0Connection.java

@@ -1,5 +1,6 @@
 package com.conveyal.datatools.manager.auth;
 
+import com.auth0.jwt.JWTExpiredException;
 import com.auth0.jwt.JWTVerifier;
 import com.auth0.jwt.pem.PemReader;
 import com.conveyal.datatools.manager.DataManager;
@@ -90,6 +91,9 @@ public static void checkUser(Request req) {
             // The user attribute is used on the server side to check user permissions and does not have all of the
             // fields that the raw Auth0 profile string does.
             req.attribute("user", profile);
+        } catch (JWTExpiredException e) {
+            LOG.warn("JWT token has expired for user.");
+            logMessageAndHalt(req, 401, "User's authentication token has expired. Please re-login.");
         } catch (Exception e) {
             LOG.warn("Login failed to verify with our authorization provider.", e);
             logMessageAndHalt(req, 401, "Could not verify user's token");

src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java

@@ -7,6 +7,7 @@
 import com.amazonaws.services.ec2.model.Filter;
 import com.amazonaws.services.ec2.model.Instance;
 import com.amazonaws.services.ec2.model.Reservation;
+import com.amazonaws.services.s3.AmazonS3URI;
 import com.conveyal.datatools.common.utils.SparkUtils;
 import com.conveyal.datatools.manager.DataManager;
 import com.conveyal.datatools.manager.auth.Auth0UserProfile;
@@ -18,8 +19,10 @@
 import com.conveyal.datatools.manager.models.JsonViews;
 import com.conveyal.datatools.manager.models.OtpServer;
 import com.conveyal.datatools.manager.models.Project;
+import com.conveyal.datatools.manager.persistence.FeedStore;
 import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.datatools.manager.utils.json.JsonManager;
+import com.mongodb.client.FindIterable;
 import org.bson.Document;
 import org.eclipse.jetty.http.HttpStatus;
 import org.slf4j.Logger;
@@ -38,6 +41,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static com.conveyal.datatools.common.utils.S3Utils.downloadFromS3;
 import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt;
 import static spark.Spark.delete;
 import static spark.Spark.get;
@@ -84,6 +88,46 @@ private static Deployment deleteDeployment (Request req, Response res) {
         return deployment;
     }
 
+    /**
+     * HTTP endpoint for downloading a build artifact (e.g., otp build log or Graph.obj) from S3.
+     */
+    private static String downloadBuildArtifact (Request req, Response res) {
+        Deployment deployment = checkDeploymentPermissions(req, res);
+        DeployJob.DeploySummary summaryToDownload = null;
+        String uriString = null;
+        // If a jobId query param is provided, find the matching job summary.
+        String jobId = req.queryParams("jobId");
+        if (jobId != null) {
+            for (DeployJob.DeploySummary summary : deployment.deployJobSummaries) {
+                if (summary.jobId.equals(jobId)) {
+                    summaryToDownload = summary;
+                    break;
+                }
+            }
+        } else {
+            summaryToDownload = deployment.latest();
+        }
+        if (summaryToDownload == null) {
+            // Try to construct the URI string
+            OtpServer server = Persistence.servers.getById(deployment.deployedTo);
+            if (server == null) {
+                uriString = String.format("s3://%s/bundles/%s/%s/%s", "S3_BUCKET", deployment.projectId, deployment.id, jobId);
+                logMessageAndHalt(req, 400, "Cannot construct URI for build artifact. " + uriString);
+                return null;
+            }
+            uriString = String.format("s3://%s/bundles/%s/%s/%s", server.s3Bucket, deployment.projectId, deployment.id, jobId);
+            LOG.warn("Could not find deploy summary for job. Attempting to use {}", uriString);
+        } else {
+            uriString = summaryToDownload.buildArtifactsFolder;
+        }
+        AmazonS3URI uri = new AmazonS3URI(uriString);
+        String filename = req.queryParams("filename");
+        if (filename == null) {
+            logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Must provide filename query param for build artifact.");
+        }
+        return downloadFromS3(FeedStore.s3Client, uri.getBucket(), String.join("/", uri.getKey(), filename), false, res);
+    }
+
     /**
      * Download all of the GTFS files in the feed.
      *
@@ -266,6 +310,9 @@ public static List<EC2InstanceSummary> fetchEC2InstanceSummaries(Filter... filte
         return fetchEC2Instances(filters).stream().map(EC2InstanceSummary::new).collect(Collectors.toList());
     }
 
+    /**
+     * Fetch EC2 instances from AWS that match the provided set of filters (e.g., tags, instance ID, or other properties).
+     */
     public static List<Instance> fetchEC2Instances(Filter... filters) {
         List<Instance> instances = new ArrayList<>();
         DescribeInstancesRequest request = new DescribeInstancesRequest().withFilters(filters);
@@ -291,7 +338,10 @@ private static String deploy (Request req, Response res) {
             logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid");
         // Get server by ID
         OtpServer otpServer = Persistence.servers.getById(target);
-        if (otpServer == null) logMessageAndHalt(req, 400, "Must provide valid OTP server target ID.");
+        if (otpServer == null) {
+            logMessageAndHalt(req, 400, "Must provide valid OTP server target ID.");
+            return null;
+        }
 
         // Check that permissions of user allow them to deploy to target.
         boolean isProjectAdmin = userProfile.canAdministerProject(deployment.projectId, deployment.organizationId());
@@ -350,7 +400,11 @@ public static void register (String apiPrefix) {
         }), json::write);
         options(apiPrefix + "secure/deployments", (q, s) -> "");
         get(apiPrefix + "secure/deployments/:id/download", DeploymentController::downloadDeployment);
+        get(apiPrefix + "secure/deployments/:id/artifact", DeploymentController::downloadBuildArtifact);
         get(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::fetchEC2InstanceSummaries, json::write);
+        // TODO: In the future, we may have need for terminating a single EC2 instance. For now, an admin using the AWS
+        //  console should suffice.
+//        delete(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::terminateEC2Instance, json::write);
         get(apiPrefix + "secure/deployments/:id", DeploymentController::getDeployment, json::write);
         delete(apiPrefix + "secure/deployments/:id", DeploymentController::deleteDeployment, json::write);
         get(apiPrefix + "secure/deployments", DeploymentController::getAllDeployments, json::write);

src/main/java/com/conveyal/datatools/manager/controllers/api/OrganizationController.java

@@ -68,8 +68,8 @@ public static Organization createOrganization (Request req, Response res) {
 
     public static Organization updateOrganization (Request req, Response res) throws IOException {
         String organizationId = req.params("id");
-        requestOrganizationById(req);
-        Organization organization = Persistence.organizations.update(organizationId, req.body());
+        Organization updatedOrganization = requestOrganizationById(req);
+        Persistence.organizations.replace(organizationId, updatedOrganization);
 
         // FIXME: Add back in hook after organization is updated.
         // See https://github.com/catalogueglobal/datatools-server/issues/111
@@ -101,7 +101,7 @@ public static Organization updateOrganization (Request req, Response res) throws
 //            p.save();
 //        }
 
-        return organization;
+        return updatedOrganization;
     }
 
     public static Organization deleteOrganization (Request req, Response res) {