From ac0b00c585e62bbb9e8a1b358234dc8f07fb3688 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 7 Aug 2018 16:43:58 -0400 Subject: [PATCH 01/42] WIP spin up EC2 (no user data) --- .../datatools/manager/jobs/DeployJob.java | 63 +++++++++++++++++++ .../datatools/manager/models/OtpServer.java | 1 + 2 files changed, 64 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 2f421e857..3f143f9d7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -2,6 +2,13 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.event.ProgressListener; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.CreateTagsRequest; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.RunInstancesRequest; +import com.amazonaws.services.ec2.model.RunInstancesResult; +import com.amazonaws.services.ec2.model.Tag; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @@ -24,11 +31,13 @@ import java.util.Scanner; import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.codec.binary.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -151,6 +160,19 @@ public void jobLogic () { status.uploadingS3 = false; } + if (otpServer.createServer) { + if ("true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled"))) { + createServer(); + // If creating a new server, there is no need to deploy to an existing one. + return; + } else { + String message = "Cannot complete deployment. EC2 deployment disabled in server configuration."; + LOG.error(message); + status.fail(message); + return; + } + } + // If there are no OTP targets (i.e. we're only deploying to S3), we're done. if(otpServer.internalUrl == null) { status.completed = true; @@ -323,6 +345,47 @@ public void jobFinished () { NotifyUsersForSubscriptionJob.createNotification("deployment-updated", deployment.id, message); } + public void createServer () { + try { + // CONNECT TO EC2 + AmazonEC2 ec2 = new AmazonEC2Client(); + ec2.setEndpoint("ec2.us-east-1.amazonaws.com"); + // User data should contain info about: + // 1. Downloading GTFS/OSM info (s3) + // 2. Time to live until shutdown/termination (for test servers) + // 3. Hosting / nginx + String myUserData = "hello"; + // CREATE EC2 INSTANCES + RunInstancesRequest runInstancesRequest = new RunInstancesRequest() + .withInstanceType("t1.micro") + .withMinCount(1) + .withSubnetId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.subnet")) + .withImageId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ami")) + .withKeyName(DataManager.getConfigPropertyAsText("modules.deployment.ec2.keyName")) + .withInstanceInitiatedShutdownBehavior("terminate") + .withMaxCount(1) + .withSecurityGroupIds(DataManager.getConfigPropertyAsText("modules.deployment.ec2.securityGroup")) + .withUserData(Base64.encodeBase64String(myUserData.getBytes())); + + RunInstancesResult runInstances = ec2.runInstances(runInstancesRequest); + + // TAG EC2 INSTANCES + List instances = runInstances.getReservation().getInstances(); + int idx = 1; + for (Instance instance : instances) { + CreateTagsRequest createTagsRequest = new CreateTagsRequest(); + createTagsRequest.withResources(instance.getInstanceId()) // + .withTags(new Tag("Name", "otp-" + idx)); + ec2.createTags(createTagsRequest); + + idx++; + } + } catch (Exception e) { + LOG.error("Could not deploy to EC2 server", e); + status.fail("Could not deploy to EC2 server", e); + } + } + /** * Represents the current status of this job. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java index b9065ce4d..c60ada3a5 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java @@ -14,6 +14,7 @@ public class OtpServer implements Serializable { public Boolean admin; public String s3Bucket; public String s3Credentials; + public boolean createServer; /** * Convert the name field into a string with no special characters. From 560c4c6b8179e5690443ea9148af395ed47ef07e Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 14:27:50 -0500 Subject: [PATCH 02/42] refactor(snapshot): remove legacy MapDB-based snapshot jobs --- .../jobs/ProcessGtfsSnapshotExport.java | 94 --- .../editor/jobs/ProcessGtfsSnapshotMerge.java | 537 ------------------ .../jobs/ProcessGtfsSnapshotUpload.java | 73 --- .../datatools/editor/models/Snapshot.java | 27 - .../editor/ProcessGtfsSnapshotMergeTest.java | 176 ------ 5 files changed, 907 deletions(-) delete mode 100755 src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotExport.java delete mode 100755 src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotMerge.java delete mode 100755 src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotUpload.java delete mode 100644 src/test/java/com/conveyal/datatools/editor/ProcessGtfsSnapshotMergeTest.java diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotExport.java b/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotExport.java deleted file mode 100755 index d4450cc59..000000000 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotExport.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.conveyal.datatools.editor.jobs; - -import com.beust.jcommander.internal.Lists; -import com.conveyal.datatools.common.status.MonitorableJob; -import com.conveyal.gtfs.GTFSFeed; -import com.conveyal.datatools.editor.datastore.FeedTx; -import com.conveyal.datatools.editor.datastore.GlobalTx; -import com.conveyal.datatools.editor.datastore.VersionedDataStore; -import com.conveyal.datatools.editor.models.Snapshot; - -import java.time.LocalDate; - -import org.mapdb.Fun.Tuple2; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.Arrays; -import java.util.Collection; - -public class ProcessGtfsSnapshotExport extends MonitorableJob { - public static final Logger LOG = LoggerFactory.getLogger(ProcessGtfsSnapshotExport.class); - private Collection> snapshots; - private File output; -// private LocalDate startDate; -// private LocalDate endDate; - - /** Export the named snapshots to GTFS */ - public ProcessGtfsSnapshotExport(Collection> snapshots, File output, LocalDate startDate, LocalDate endDate) { - super("application", "Exporting snapshots to GTFS", JobType.PROCESS_SNAPSHOT_EXPORT); - this.snapshots = snapshots; - this.output = output; -// this.startDate = startDate; -// this.endDate = endDate; - } - - /** - * Export the master branch of the named feeds to GTFS. The boolean variable can be either true or false, it is only to make this - * method have a different erasure from the other - */ - public ProcessGtfsSnapshotExport(Collection agencies, File output, LocalDate startDate, LocalDate endDate, boolean isagency) { - super("application", "Exporting snapshots to GTFS", JobType.PROCESS_SNAPSHOT_EXPORT); - this.snapshots = Lists.newArrayList(agencies.size()); - - for (String agency : agencies) { - // leaving version null will cause master to be used - this.snapshots.add(new Tuple2(agency, null)); - } - - this.output = output; -// this.startDate = startDate; -// this.endDate = endDate; - } - - /** - * Export this snapshot to GTFS, using the validity range in the snapshot. - */ - public ProcessGtfsSnapshotExport (Snapshot snapshot, File output) { - this(Arrays.asList(new Tuple2[] { snapshot.id }), output, snapshot.validFrom, snapshot.validTo); - } - - @Override - public void jobLogic() { - GTFSFeed feed = null; - - GlobalTx gtx = VersionedDataStore.getGlobalTx(); - FeedTx feedTx = null; - - try { - for (Tuple2 ssid : snapshots) { - String feedId = ssid.a; - - // retrieveById present feed database if no snapshot version provided - if (ssid.b == null) { - feedTx = VersionedDataStore.getFeedTx(feedId); - } - // else retrieveById snapshot version data - else { - feedTx = VersionedDataStore.getFeedTx(feedId, ssid.b); - } - feed = feedTx.toGTFSFeed(false); - } - feed.toFile(output.getAbsolutePath()); - } finally { - gtx.rollbackIfOpen(); - if (feedTx != null) feedTx.rollbackIfOpen(); - } - } - - public static int toGtfsDate (LocalDate date) { - return date.getYear() * 10000 + date.getMonthValue() * 100 + date.getDayOfMonth(); - } -} - diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotMerge.java b/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotMerge.java deleted file mode 100755 index 23816dc5f..000000000 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotMerge.java +++ /dev/null @@ -1,537 +0,0 @@ -package com.conveyal.datatools.editor.jobs; - -import com.conveyal.datatools.common.status.MonitorableJob; -import com.conveyal.datatools.editor.datastore.FeedTx; -import com.conveyal.datatools.editor.models.Snapshot; -import com.conveyal.datatools.editor.models.transit.Agency; -import com.conveyal.datatools.editor.models.transit.EditorFeed; -import com.conveyal.datatools.editor.models.transit.GtfsRouteType; -import com.conveyal.datatools.editor.models.transit.Route; -import com.conveyal.datatools.editor.models.transit.RouteType; -import com.conveyal.datatools.editor.models.transit.ServiceCalendar; -import com.conveyal.datatools.editor.models.transit.Stop; -import com.conveyal.datatools.manager.models.FeedVersion; -import com.conveyal.gtfs.loader.Feed; -import com.google.common.collect.Maps; -import com.vividsolutions.jts.geom.Envelope; -import com.vividsolutions.jts.geom.GeometryFactory; -import com.vividsolutions.jts.geom.PrecisionModel; -import com.conveyal.datatools.editor.datastore.GlobalTx; -import com.conveyal.datatools.editor.datastore.VersionedDataStore; -import gnu.trove.map.TIntObjectMap; -import gnu.trove.map.hash.TIntObjectHashMap; - -import java.awt.geom.Rectangle2D; - -import org.mapdb.Fun.Tuple2; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -public class ProcessGtfsSnapshotMerge extends MonitorableJob { - public static final Logger LOG = LoggerFactory.getLogger(ProcessGtfsSnapshotMerge.class); - /** map from GTFS agency IDs to Agencies */ - private Map agencyIdMap = new HashMap<>(); - private Map routeIdMap = new HashMap<>(); - /** map from (gtfs stop ID, database agency ID) -> stop */ - private Map, Stop> stopIdMap = Maps.newHashMap(); - private TIntObjectMap routeTypeIdMap = new TIntObjectHashMap<>(); - - private Feed inputFeedTables; - private EditorFeed editorFeed; - - public FeedVersion feedVersion; - - /*public ProcessGtfsSnapshotMerge (File gtfsFile) { - this(gtfsFile, null); - }*/ - - public ProcessGtfsSnapshotMerge (FeedVersion feedVersion, String owner) { - super(owner, "Creating snapshot for " + feedVersion.parentFeedSource().name, JobType.PROCESS_SNAPSHOT_MERGE); - this.feedVersion = feedVersion; - status.update(false, "Waiting to begin job...", 0); - LOG.info("GTFS Snapshot Merge for feedVersion {}", feedVersion.id); - } - - public void jobLogic () { - long agencyCount = 0; - long routeCount = 0; - long stopCount = 0; - long stopTimeCount = 0; - long tripCount = 0; - long shapePointCount = 0; - long serviceCalendarCount = 0; - long fareCount = 0; - - GlobalTx gtx = VersionedDataStore.getGlobalTx(); - - // create a new feed based on this version - FeedTx feedTx = VersionedDataStore.getFeedTx(feedVersion.feedSourceId); - - editorFeed = new EditorFeed(); - editorFeed.setId(feedVersion.feedSourceId); - Rectangle2D bounds = feedVersion.validationResult.fullBounds.toRectangle2D(); - if (bounds != null) { - editorFeed.defaultLat = bounds.getCenterY(); - editorFeed.defaultLon = bounds.getCenterX(); - } - - - try { - synchronized (status) { - status.message = "Wiping old data..."; - status.percentComplete = 2; - } - // clear the existing data - for(String key : feedTx.agencies.keySet()) feedTx.agencies.remove(key); - for(String key : feedTx.routes.keySet()) feedTx.routes.remove(key); - for(String key : feedTx.stops.keySet()) feedTx.stops.remove(key); - for(String key : feedTx.calendars.keySet()) feedTx.calendars.remove(key); - for(String key : feedTx.exceptions.keySet()) feedTx.exceptions.remove(key); - for(String key : feedTx.fares.keySet()) feedTx.fares.remove(key); - for(String key : feedTx.tripPatterns.keySet()) feedTx.tripPatterns.remove(key); - for(String key : feedTx.trips.keySet()) feedTx.trips.remove(key); - LOG.info("Cleared old data"); - - synchronized (status) { - status.message = "Loading GTFS file..."; - status.percentComplete = 5; - } - - // retrieveById Feed connection to SQL tables for the feed version - inputFeedTables = feedVersion.retrieveFeed(); - if(inputFeedTables == null) return; - - LOG.info("GtfsImporter: importing feed..."); - synchronized (status) { - status.message = "Beginning feed import..."; - status.percentComplete = 8; - } - // load feed_info.txt - // FIXME add back in feed info!! -// if(inputFeedTables.feedInfo.size() > 0) { -// FeedInfo feedInfo = input.feedInfo.values().iterator().next(); -// editorFeed.feedPublisherName = feedInfo.feed_publisher_name; -// editorFeed.feedPublisherUrl = feedInfo.feed_publisher_url; -// editorFeed.feedLang = feedInfo.feed_lang; -// editorFeed.feedEndDate = feedInfo.feed_end_date; -// editorFeed.feedStartDate = feedInfo.feed_start_date; -// editorFeed.feedVersion = feedInfo.feed_version; -// } - gtx.feeds.put(feedVersion.feedSourceId, editorFeed); - - // load the GTFS agencies - Iterator agencyIterator = inputFeedTables.agencies.iterator(); - while (agencyIterator.hasNext()) { - com.conveyal.gtfs.model.Agency gtfsAgency = agencyIterator.next(); - Agency agency = new Agency(gtfsAgency, editorFeed); - - // don't save the agency until we've come up with the stop centroid, below. - agencyCount++; - - // we do want to use the modified agency ID here, because everything that refers to it has a reference - // to the agency object we updated. - feedTx.agencies.put(agency.id, agency); - agencyIdMap.put(gtfsAgency.agency_id, agency); - } - synchronized (status) { - status.message = "Agencies loaded: " + agencyCount; - status.percentComplete = 10; - } - LOG.info("Agencies loaded: " + agencyCount); - - LOG.info("GtfsImporter: importing stops..."); - synchronized (status) { - status.message = "Importing stops..."; - status.percentComplete = 15; - } - // TODO: remove stop ownership inference entirely? - // infer agency ownership of stops, if there are multiple feeds -// SortedSet> stopsByAgency = inferAgencyStopOwnership(); - - // build agency centroids as we go - // note that these are not actually centroids, but the center of the extent of the stops . . . - Map stopEnvelopes = Maps.newHashMap(); - - for (Agency agency : agencyIdMap.values()) { - stopEnvelopes.put(agency.id, new Envelope()); - } - - GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(), 4326); - for (com.conveyal.gtfs.model.Stop gtfsStop : inputFeedTables.stops) { - Stop stop = new Stop(gtfsStop, geometryFactory, editorFeed); - feedTx.stops.put(stop.id, stop); - stopIdMap.put(new Tuple2(gtfsStop.stop_id, editorFeed.id), stop); - stopCount++; - } - - LOG.info("Stops loaded: " + stopCount); - synchronized (status) { - status.message = "Stops loaded: " + stopCount; - status.percentComplete = 25; - } - LOG.info("GtfsImporter: importing routes..."); - synchronized (status) { - status.message = "Importing routes..."; - status.percentComplete = 30; - } - // import routes - for (com.conveyal.gtfs.model.Route gtfsRoute : inputFeedTables.routes) { - Agency agency = agencyIdMap.get(gtfsRoute.agency_id); - - if (!routeTypeIdMap.containsKey(gtfsRoute.route_type)) { - RouteType rt = new RouteType(); - rt.gtfsRouteType = GtfsRouteType.fromGtfs(gtfsRoute.route_type); - gtx.routeTypes.put(rt.id, rt); - routeTypeIdMap.put(gtfsRoute.route_type, rt.id); - } - - Route route = new Route(gtfsRoute, editorFeed, agency); - - feedTx.routes.put(route.id, route); - routeIdMap.put(gtfsRoute.route_id, route); - routeCount++; - } - - LOG.info("Routes loaded: " + routeCount); - synchronized (status) { - status.message = "Routes loaded: " + routeCount; - status.percentComplete = 35; - } - - LOG.info("GtfsImporter: importing Service Calendars..."); - synchronized (status) { - status.message = "Importing service calendars..."; - status.percentComplete = 38; - } - // we don't put service calendars in the database just yet, because we don't know what agency they're associated with - // we copy them into the agency database as needed - // GTFS service ID -> ServiceCalendar - Map calendars = Maps.newHashMap(); - - // FIXME: add back in services! -// for (Service svc : input.services.values()) { -// -// ServiceCalendar cal; -// -// if (svc.calendar != null) { -// // easy case: don't have to infer anything! -// cal = new ServiceCalendar(svc.calendar, feed); -// } else { -// // infer a calendar -// // number of mondays, etc. that this calendar is active -// int monday, tuesday, wednesday, thursday, friday, saturday, sunday; -// monday = tuesday = wednesday = thursday = friday = saturday = sunday = 0; -// LocalDate startDate = null; -// LocalDate endDate = null; -// -// for (CalendarDate cd : svc.calendar_dates.values()) { -// if (cd.exception_type == 2) -// continue; -// -// if (startDate == null || cd.date.isBefore(startDate)) -// startDate = cd.date; -// -// if (endDate == null || cd.date.isAfter(endDate)) -// endDate = cd.date; -// -// int dayOfWeek = cd.date.getDayOfWeek().getValue(); -// -// switch (dayOfWeek) { -// case DateTimeConstants.MONDAY: -// monday++; -// break; -// case DateTimeConstants.TUESDAY: -// tuesday++; -// break; -// case DateTimeConstants.WEDNESDAY: -// wednesday++; -// break; -// case DateTimeConstants.THURSDAY: -// thursday++; -// break; -// case DateTimeConstants.FRIDAY: -// friday++; -// break; -// case DateTimeConstants.SATURDAY: -// saturday++; -// break; -// case DateTimeConstants.SUNDAY: -// sunday++; -// break; -// } -// } -// -// // infer the calendar. if there is service on more than half as many as the maximum number of -// // a particular day that has service, assume that day has service in general. -// int maxService = Ints.max(monday, tuesday, wednesday, thursday, friday, saturday, sunday); -// -// cal = new ServiceCalendar(); -// cal.feedId = feed.id; -// -// if (startDate == null) { -// // no service whatsoever -// LOG.warn("Service ID " + svc.service_id + " has no service whatsoever"); -// startDate = LocalDate.now().minusMonths(1); -// endDate = startDate.plusYears(1); -// cal.monday = cal.tuesday = cal.wednesday = cal.thursday = cal.friday = cal.saturday = cal.sunday = false; -// } -// else { -// // infer parameters -// -// int threshold = (int) Math.round(Math.ceil((double) maxService / 2)); -// -// cal.monday = monday >= threshold; -// cal.tuesday = tuesday >= threshold; -// cal.wednesday = wednesday >= threshold; -// cal.thursday = thursday >= threshold; -// cal.friday = friday >= threshold; -// cal.saturday = saturday >= threshold; -// cal.sunday = sunday >= threshold; -// -// cal.startDate = startDate; -// cal.endDate = endDate; -// } -// -// cal.inferName(); -// cal.gtfsServiceId = svc.service_id; -// } -// -// calendars.put(svc.service_id, cal); -// -// serviceCalendarCount++; -// } - - LOG.info("Service calendars loaded: " + serviceCalendarCount); - synchronized (status) { - status.message = "Service calendars loaded: " + serviceCalendarCount; - status.percentComplete = 45; - } - LOG.info("GtfsImporter: importing trips..."); - synchronized (status) { - status.message = "Importing trips..."; - status.percentComplete = 50; - } - // FIXME need to load patterns and trips - // import trips, stop times and patterns all at once -// Map patterns = input.patterns; -// Set processedTrips = new HashSet<>(); -// for (Entry pattern : patterns.entrySet()) { -// // it is possible, though unlikely, for two routes to have the same stopping pattern -// // we want to ensure they retrieveById different trip patterns -// Map tripPatternsByRoute = Maps.newHashMap(); -// for (String tripId : pattern.getValue().associatedTrips) { -// -// // TODO: figure out why trips are being added twice. This check prevents that. -// if (processedTrips.contains(tripId)) { -// continue; -// } -// synchronized (status) { -// status.message = "Importing trips... (id: " + tripId + ") " + tripCount + "/" + input.trips.size(); -// status.percentComplete = 50 + 45 * tripCount / input.trips.size(); -// } -// com.conveyal.gtfs.model.Trip gtfsTrip = input.trips.retrieveById(tripId); -// -// if (!tripPatternsByRoute.containsKey(gtfsTrip.route_id)) { -// TripPattern pat = createTripPatternFromTrip(gtfsTrip, feedTx); -// feedTx.tripPatterns.put(pat.id, pat); -// tripPatternsByRoute.put(gtfsTrip.route_id, pat); -// } -// -// // there is more than one pattern per route, but this map is specific to only this pattern -// // generally it will contain exactly one entry, unless there are two routes with identical -// // stopping patterns. -// // (in DC, suppose there were trips on both the E2/weekday and E3/weekend from Friendship Heights -// // that short-turned at Missouri and 3rd). -// TripPattern pat = tripPatternsByRoute.retrieveById(gtfsTrip.route_id); -// -// ServiceCalendar cal = calendars.retrieveById(gtfsTrip.service_id); -// -// // if the service calendar has not yet been imported, import it -// if (feedTx.calendars != null && !feedTx.calendars.containsKey(cal.id)) { -// // no need to clone as they are going into completely separate mapdbs -// feedTx.calendars.put(cal.id, cal); -// } -// -// Trip trip = new Trip(gtfsTrip, routeIdMap.retrieveById(gtfsTrip.route_id), pat, cal); -// -// // TODO: query ordered stopTimes for a given trip id -// // FIXME: add back in stopTimes -// Collection stopTimes = new ArrayList<>(); -// input.stopTimes.subMap(new Tuple2(gtfsTrip.trip_id, null), new Tuple2(gtfsTrip.trip_id, Fun.HI)).values(); -// -// for (com.conveyal.gtfs.model.StopTime st : stopTimes) { -// trip.stopTimes.add(new StopTime(st, stopIdMap.retrieveById(new Tuple2<>(st.stop_id, feed.id)).id)); -// stopTimeCount++; -// } -// -// feedTx.trips.put(trip.id, trip); -// processedTrips.add(tripId); -// tripCount++; -// -// // FIXME add back in total number of trips for QC -// if (tripCount % 1000 == 0) { -// LOG.info("Loaded {} / {} trips", tripCount); // input.trips.size() -// } -// } -// } - - LOG.info("Trips loaded: " + tripCount); - synchronized (status) { - status.message = "Trips loaded: " + tripCount; - status.percentComplete = 90; - } - - LOG.info("GtfsImporter: importing fares..."); - // FIXME add in fares -// Map fares = input.fares; -// for (com.conveyal.gtfs.model.Fare f : fares.values()) { -// Fare fare = new Fare(f.fare_attribute, f.fare_rules, feed); -// feedTx.fares.put(fare.id, fare); -// fareCount++; -// } - LOG.info("Fares loaded: " + fareCount); - synchronized (status) { - status.message = "Fares loaded: " + fareCount; - status.percentComplete = 92; - } - LOG.info("Saving snapshot..."); - synchronized (status) { - status.message = "Saving snapshot..."; - status.percentComplete = 95; - } - // commit the feed TXs first, so that we have orphaned data rather than inconsistent data on a commit failure - feedTx.commit(); - gtx.commit(); - Snapshot.deactivateSnapshots(feedVersion.feedSourceId, null); - // create an initial snapshot for this FeedVersion - Snapshot snapshot = VersionedDataStore.takeSnapshot(editorFeed.id, feedVersion.id, "Snapshot of " + feedVersion.name, "none"); - - - LOG.info("Imported GTFS file: " + agencyCount + " agencies; " + routeCount + " routes;" + stopCount + " stops; " + stopTimeCount + " stopTimes; " + tripCount + " trips;" + shapePointCount + " shapePoints"); - synchronized (status) { - status.message = "Import complete!"; - status.percentComplete = 100; - } - } - catch (Exception e) { - e.printStackTrace(); - synchronized (status) { - status.message = "Failed to process GTFS snapshot."; - status.error = true; - } - } - finally { - feedTx.rollbackIfOpen(); - gtx.rollbackIfOpen(); - - // FIXME: anything we need to do at the end of using Feed? -// inputFeedTables.close(); - - } - } - - /** infer the ownership of stops based on what stops there - * Returns a set of tuples stop ID, agency ID with GTFS IDs */ -// private SortedSet> inferAgencyStopOwnership() { -// SortedSet> ret = Sets.newTreeSet(); -// -// for (com.conveyal.gtfs.model.StopTime st : input.stop_times.values()) { -// String stopId = st.stop_id; -// com.conveyal.gtfs.model.Trip trip = input.trips.retrieveById(st.trip_id); -// if (trip != null) { -// String routeId = trip.route_id; -// String agencyId = input.routes.retrieveById(routeId).agency_id; -// Tuple2 key = new Tuple2(stopId, agencyId); -// ret.add(key); -// } -// } -// -// return ret; -// } - - /** - * Create a trip pattern from the given trip. - * Neither the TripPattern nor the TripPatternStops are saved. - */ -// public TripPattern createTripPatternFromTrip (com.conveyal.gtfs.model.Trip gtfsTrip, FeedTx tx) { -// TripPattern patt = new TripPattern(); -// com.conveyal.gtfs.model.Route gtfsRoute = input.routes.retrieveById(gtfsTrip.route_id); -// patt.routeId = routeIdMap.retrieveById(gtfsTrip.route_id).id; -// patt.feedId = feed.id; -// -// String patternId = input.tripPatternMap.retrieveById(gtfsTrip.trip_id); -// Pattern gtfsPattern = input.patterns.retrieveById(patternId); -// patt.shape = gtfsPattern.geometry; -// patt.id = gtfsPattern.pattern_id; -// -// patt.patternStops = new ArrayList<>(); -// patt.patternDirection = TripDirection.fromGtfs(gtfsTrip.direction_id); -// -// com.conveyal.gtfs.model.StopTime[] stopTimes = -// input.stop_times.subMap(new Tuple2(gtfsTrip.trip_id, 0), new Tuple2(gtfsTrip.trip_id, Fun.HI)).values().toArray(new com.conveyal.gtfs.model.StopTime[0]); -// -// if (gtfsTrip.trip_headsign != null && !gtfsTrip.trip_headsign.isEmpty()) -// patt.name = gtfsTrip.trip_headsign; -// else -// patt.name = gtfsPattern.name; -// -// for (com.conveyal.gtfs.model.StopTime st : stopTimes) { -// TripPatternStop tps = new TripPatternStop(); -// -// Stop stop = stopIdMap.retrieveById(new Tuple2(st.stop_id, patt.feedId)); -// tps.stopId = stop.id; -// -// // set timepoint according to first gtfs value and then whether arrival and departure times are present -// if (st.timepoint != Entity.INT_MISSING) -// tps.timepoint = st.timepoint == 1; -// else if (st.arrival_time != Entity.INT_MISSING && st.departure_time != Entity.INT_MISSING) { -// tps.timepoint = true; -// } -// else -// tps.timepoint = false; -// -// if (st.departure_time != Entity.INT_MISSING && st.arrival_time != Entity.INT_MISSING) -// tps.defaultDwellTime = st.departure_time - st.arrival_time; -// else -// tps.defaultDwellTime = 0; -// -// patt.patternStops.add(tps); -// } -// -// patt.calcShapeDistTraveled(tx); -// -// // infer travel times -// if (stopTimes.length >= 2) { -// int startOfBlock = 0; -// // start at one because the first stop has no travel time -// // but don't put nulls in the data -// patt.patternStops.retrieveById(0).defaultTravelTime = 0; -// for (int i = 1; i < stopTimes.length; i++) { -// com.conveyal.gtfs.model.StopTime current = stopTimes[i]; -// -// if (current.arrival_time != Entity.INT_MISSING) { -// // interpolate times -// -// int timeSinceLastSpecifiedTime = current.arrival_time - stopTimes[startOfBlock].departure_time; -// -// double blockLength = patt.patternStops.retrieveById(i).shapeDistTraveled - patt.patternStops.retrieveById(startOfBlock).shapeDistTraveled; -// -// // go back over all of the interpolated stop times and interpolate them -// for (int j = startOfBlock + 1; j <= i; j++) { -// TripPatternStop tps = patt.patternStops.retrieveById(j); -// double distFromLastStop = patt.patternStops.retrieveById(j).shapeDistTraveled - patt.patternStops.retrieveById(j - 1).shapeDistTraveled; -// tps.defaultTravelTime = (int) Math.round(timeSinceLastSpecifiedTime * distFromLastStop / blockLength); -// } -// -// startOfBlock = i; -// } -// } -// } -// -// return patt; -// } - -} - diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotUpload.java b/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotUpload.java deleted file mode 100755 index c30be3030..000000000 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ProcessGtfsSnapshotUpload.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.conveyal.datatools.editor.jobs; - -//import play.jobs.Job; - -public class ProcessGtfsSnapshotUpload implements Runnable { - @Override - public void run() { - - } - /* - private Long _gtfsSnapshotMergeId; - - private Map agencyIdMap = new HashMap(); - - public ProcessGtfsSnapshotUpload(Long gtfsSnapshotMergeId) { - this._gtfsSnapshotMergeId = gtfsSnapshotMergeId; - } - - public void doJob() { - - GtfsSnapshotMerge snapshotMerge = null; - while(snapshotMerge == null) - { - snapshotMerge = GtfsSnapshotMerge.findById(this._gtfsSnapshotMergeId); - LOG.warn("Waiting for snapshotMerge to save..."); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - GtfsReader reader = new GtfsReader(); - GtfsDaoImpl store = new GtfsDaoImpl(); - - Long agencyCount = new Long(0); - - try { - - File gtfsFile = new File(Play.configuration.getProperty("application.publicGtfsDataDirectory"), snapshotMerge.snapshot.getFilename()); - - reader.setInputLocation(gtfsFile); - reader.setEntityStore(store); - reader.run(); - - LOG.info("GtfsImporter: listing feeds..."); - - for (org.onebusaway.gtfs.model.Agency gtfsAgency : reader.getAgencies()) { - - GtfsAgency agency = new GtfsAgency(gtfsAgency); - agency.snapshot = snapshotMerge.snapshot; - agency.save(); - - } - - snapshotMerge.snapshot.agencyCount = store.getAllAgencies().size(); - snapshotMerge.snapshot.routeCount = store.getAllRoutes().size(); - snapshotMerge.snapshot.stopCount = store.getAllStops().size(); - snapshotMerge.snapshot.tripCount = store.getAllTrips().size(); - - snapshotMerge.snapshot.save(); - - } - catch (Exception e) { - - LOG.error(e.toString()); - - snapshotMerge.failed(e.toString()); - } - }*/ -} - diff --git a/src/main/java/com/conveyal/datatools/editor/models/Snapshot.java b/src/main/java/com/conveyal/datatools/editor/models/Snapshot.java index ada896941..99d71a6c2 100644 --- a/src/main/java/com/conveyal/datatools/editor/models/Snapshot.java +++ b/src/main/java/com/conveyal/datatools/editor/models/Snapshot.java @@ -2,7 +2,6 @@ import com.conveyal.datatools.editor.datastore.GlobalTx; import com.conveyal.datatools.editor.datastore.VersionedDataStore; -import com.conveyal.datatools.editor.jobs.ProcessGtfsSnapshotExport; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -89,32 +88,6 @@ public String generateFileName () { return this.feedId + "_" + this.snapshotTime + ".zip"; } - /** Write snapshot to disk as GTFS */ - public static boolean writeSnapshotAsGtfs (Tuple2 decodedId, File outFile) { - GlobalTx gtx = VersionedDataStore.getGlobalTx(); - Snapshot local; - try { - if (!gtx.snapshots.containsKey(decodedId)) { - return false; - } - local = gtx.snapshots.get(decodedId); - new ProcessGtfsSnapshotExport(local, outFile).run(); - } finally { - gtx.rollbackIfOpen(); - } - return true; - } - - public static boolean writeSnapshotAsGtfs (String id, File outFile) { - Tuple2 decodedId; - try { - decodedId = JacksonSerializers.Tuple2IntDeserializer.deserialize(id); - } catch (IOException e1) { - return false; - } - return writeSnapshotAsGtfs(decodedId, outFile); - } - @JsonIgnore public static Collection getSnapshots (String feedId) { GlobalTx gtx = VersionedDataStore.getGlobalTx(); diff --git a/src/test/java/com/conveyal/datatools/editor/ProcessGtfsSnapshotMergeTest.java b/src/test/java/com/conveyal/datatools/editor/ProcessGtfsSnapshotMergeTest.java deleted file mode 100644 index 45717fb2c..000000000 --- a/src/test/java/com/conveyal/datatools/editor/ProcessGtfsSnapshotMergeTest.java +++ /dev/null @@ -1,176 +0,0 @@ -package com.conveyal.datatools.editor; - -import com.conveyal.datatools.editor.jobs.ProcessGtfsSnapshotMerge; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Created by landon on 2/24/17. - */ -public class ProcessGtfsSnapshotMergeTest { - private static final Logger LOG = LoggerFactory.getLogger(ProcessGtfsSnapshotMergeTest.class); - static ProcessGtfsSnapshotMerge snapshotMerge; - private static boolean setUpIsDone = false; - - // TODO: add back in test once editor load is working -// @Before -// public void setUp() { -// if (setUpIsDone) { -// return; -// } -// super.setUp(); -// LOG.info("ProcessGtfsSnapshotMergeTest setup"); -// -// snapshotMerge = new ProcessGtfsSnapshotMerge(super.version, "test@conveyal.com"); -// snapshotMerge.run(); -// setUpIsDone = true; -// } -// -// @Test -// public void countRoutes() { -// FeedTx feedTx = VersionedDataStore.getFeedTx(source.id); -// assertEquals(feedTx.routes.size(), 3); -// } -// -// @Test -// public void countStops() { -// FeedTx feedTx = VersionedDataStore.getFeedTx(source.id); -// assertEquals(feedTx.stops.size(), 31); -// } -// -// @Test -// public void countTrips() { -// FeedTx feedTx = VersionedDataStore.getFeedTx(source.id); -// assertEquals(feedTx.trips.size(), 252); -// } -// -// @Test -// public void countFares() { -// FeedTx feedTx = VersionedDataStore.getFeedTx(source.id); -// assertEquals(feedTx.fares.size(), 6); -// } - -// @Test -// public void duplicateStops() { -// ValidationResult result = new ValidationResult(); -// -// result = gtfsValidation1.duplicateStops(); -// Assert.assertEquals(result.invalidValues.size(), 0); -// -// -// // try duplicate stop test to confirm that stops within the buffer limit are found -// result = gtfsValidation1.duplicateStops(25.0); -// Assert.assertEquals(result.invalidValues.size(), 1); -// -// // try same test to confirm that buffers below the limit don't detect duplicates -// result = gtfsValidation1.duplicateStops(5.0); -// Assert.assertEquals(result.invalidValues.size(), 0); -// } -// -// @Test -// public void reversedTripShapes() { -// -// ValidationResult result = gtfsValidation1.listReversedTripShapes(); -// -// Assert.assertEquals(result.invalidValues.size(), 1); -// -// // try again with an unusually high distanceMultiplier value -// result = gtfsValidation1.listReversedTripShapes(50000.0); -// -// Assert.assertEquals(result.invalidValues.size(), 0); -// -// } -// -// -// @Test -// public void validateTrips() { -// ValidationResult result = gtfsValidation2.validateTrips(); -// -// Assert.assertEquals(result.invalidValues.size(), 9); -// -// } -// -// @Test -// public void completeBadGtfsTest() { -// -// GtfsDaoImpl gtfsStore = new GtfsDaoImpl(); -// -// GtfsReader gtfsReader = new GtfsReader(); -// -// File gtfsFile = new File("src/test/resources/st_gtfs_bad.zip"); -// -// try { -// -// gtfsReader.setInputLocation(gtfsFile); -// -// } catch (IOException e) { -// e.printStackTrace(); -// } -// -// gtfsReader.setEntityStore(gtfsStore); -// -// -// try { -// gtfsReader.run(); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// -// try { -// GtfsValidationService gtfsValidation = new GtfsValidationService(gtfsStore); -// -// ValidationResult results = gtfsValidation.validateRoutes(); -// results.add(gtfsValidation.validateTrips()); -// -// Assert.assertEquals(results.invalidValues.size(), 5); -// -// System.out.println(results.invalidValues.size()); -// -// } catch (Exception e) { -// e.printStackTrace(); -// } -// -// } -// -// @Test -// public void completeGoodGtfsTest() { -// -// GtfsDaoImpl gtfsStore = new GtfsDaoImpl(); -// -// GtfsReader gtfsReader = new GtfsReader(); -// -// File gtfsFile = new File("src/test/resources/st_gtfs_good.zip"); -// -// try { -// -// gtfsReader.setInputLocation(gtfsFile); -// -// } catch (IOException e) { -// e.printStackTrace(); -// } -// -// gtfsReader.setEntityStore(gtfsStore); -// -// -// try { -// gtfsReader.run(); -// } catch (Exception e) { -// e.printStackTrace(); -// } -// -// try { -// GtfsValidationService gtfsValidation = new GtfsValidationService(gtfsStore); -// -// ValidationResult results = gtfsValidation.validateRoutes(); -// results.add(gtfsValidation.validateTrips()); -// -// Assert.assertEquals(results.invalidValues.size(), 0); -// -// System.out.println(results.invalidValues.size()); -// -// } catch (Exception e) { -// e.printStackTrace(); -// } -// -// } -} From dbbd13083c24e521d0c2e9cf85d2d360352ad012 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 14:34:55 -0500 Subject: [PATCH 03/42] fix(delete): delete SQL namespace when feed version/snapshot deleted fixes #120 --- .../controllers/api/SnapshotController.java | 2 +- .../controllers/api/ProjectController.java | 5 +---- .../datatools/manager/models/FeedSource.java | 17 ++++++++++------- .../datatools/manager/models/FeedVersion.java | 2 ++ .../datatools/manager/models/Project.java | 7 ++----- .../datatools/manager/models/Snapshot.java | 19 +++++++++++++++++++ 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java index 40bbe1f09..48a0719ef 100644 --- a/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java +++ b/src/main/java/com/conveyal/datatools/editor/controllers/api/SnapshotController.java @@ -213,7 +213,7 @@ private static Snapshot deleteSnapshot(Request req, Response res) { if (snapshot == null) haltWithMessage(req, 400, "Must provide valid snapshot ID."); try { // Remove the snapshot and then renumber the snapshots - Persistence.snapshots.removeById(snapshot.id); + snapshot.delete(); feedSource.renumberSnapshots(); // FIXME Are there references that need to be removed? E.g., what if the active buffer snapshot is deleted? // FIXME delete tables from database? diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java index 86f78ca40..d506c4cc3 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java @@ -138,10 +138,7 @@ private static Project updateProject(Request req, Response res) throws IOExcepti private static Project deleteProject(Request req, Response res) { // Fetch project first to check permissions, and so we can return the deleted project after deletion. Project project = requestProjectById(req, "manage"); - boolean successfullyDeleted = project.delete(); - if (!successfullyDeleted) { - haltWithMessage(req, 400, "Did not delete project."); - } + project.delete(); return project; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index b178d6aba..d85b14346 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -4,13 +4,12 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ObjectMetadata; import com.conveyal.datatools.common.status.MonitorableJob; -import com.conveyal.datatools.editor.datastore.GlobalTx; -import com.conveyal.datatools.editor.datastore.VersionedDataStore; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.jobs.NotifyUsersForSubscriptionJob; import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HashUtils; +import com.conveyal.gtfs.GTFS; import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -30,7 +29,6 @@ import java.util.Collection; import java.util.Date; import java.util.HashMap; -import java.util.List; import java.util.Map; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; @@ -510,10 +508,16 @@ public enum FeedRetrievalMethod { * * FIXME: Use a Mongo transaction to handle the deletion of these related objects. */ - public boolean delete() { + public void delete() { try { + // Remove all feed version records for this feed source retrieveFeedVersions().forEach(FeedVersion::delete); - + // Remove all snapshot records for this feed source + retrieveSnapshots().forEach(Snapshot::delete); + // Delete active editor buffer if exists. + if (this.editorNamespace != null) { + GTFS.delete(this.editorNamespace, DataManager.GTFS_DATA_SOURCE); + } // Delete latest copy of feed source on S3. if (DataManager.useS3) { DeleteObjectsRequest delete = new DeleteObjectsRequest(DataManager.feedBucket); @@ -527,10 +531,9 @@ public boolean delete() { // editor snapshots)? // Finally, delete the feed source mongo document. - return Persistence.feedSources.removeById(this.id); + Persistence.feedSources.removeById(this.id); } catch (Exception e) { LOG.error("Could not delete feed source", e); - return false; } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index c337559eb..ab0658c5a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -384,6 +384,8 @@ public void delete() { Persistence.feedSources.update(fs.id, "{lastFetched:null}"); } feedStore.deleteFeed(id); + // Delete feed version tables in GTFS database + GTFS.delete(this.namespace, DataManager.GTFS_DATA_SOURCE); // Remove this FeedVersion from all Deployments associated with this FeedVersion's FeedSource's Project // TODO TEST THOROUGHLY THAT THIS UPDATE EXPRESSION IS CORRECT // Although outright deleting the feedVersion from deployments could be surprising and shouldn't be done anyway. diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index dd65d9850..4757b0e35 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -100,16 +100,13 @@ public Organization retrieveOrganization() { } } - public boolean delete() { + public void delete() { // FIXME: Handle this in a Mongo transaction. See https://docs.mongodb.com/master/core/transactions/#transactions-and-mongodb-drivers -// ClientSession clientSession = Persistence.startSession(); -// clientSession.startTransaction(); - // Delete each feed source in the project (which in turn deletes each feed version). retrieveProjectFeedSources().forEach(FeedSource::delete); // Delete each deployment in the project. retrieveDeployments().forEach(Deployment::delete); // Finally, delete the project. - return Persistence.projects.removeById(this.id); + Persistence.projects.removeById(this.id); } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Snapshot.java b/src/main/java/com/conveyal/datatools/manager/models/Snapshot.java index e9401979c..365853eb7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Snapshot.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Snapshot.java @@ -1,9 +1,16 @@ package com.conveyal.datatools.manager.models; +import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.GTFS; import com.conveyal.gtfs.loader.FeedLoadResult; +import com.conveyal.gtfs.util.InvalidNamespaceException; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonAlias; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.sql.SQLException; import java.util.Date; /** @@ -15,6 +22,7 @@ public class Snapshot extends Model { public static final long serialVersionUID = 1L; public static final String FEED_SOURCE_REF = "feedSourceId"; + private static final Logger LOG = LoggerFactory.getLogger(Snapshot.class); /** Is this snapshot the current snapshot - the most recently created or restored (i.e. the most current view of what's in master */ public boolean current; @@ -70,6 +78,17 @@ public Snapshot(String feedSourceId, String snapshotOf) { generateName(); } + public void delete () { + try { + // Delete snapshot tables in GTFS database + GTFS.delete(this.namespace, DataManager.GTFS_DATA_SOURCE); + // If SQL delete is successful, delete Mongo record. + Persistence.snapshots.removeById(this.id); + } catch (InvalidNamespaceException | SQLException e) { + LOG.error("Could not delete snapshot", e); + } + } + public void generateName() { this.name = "New snapshot " + new Date().toString(); } From c01eadc7fd011b7589f21a1b45b0f48a5dab9cd4 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 14:38:49 -0500 Subject: [PATCH 04/42] feature(deploy-ec2): deployment enhancements for load balancers Adds a separate controller/collection to manage OTP servers and allow for deploying to a new EC2 servers and adding them to a load balancer rather than building an OTP graph over the wire. --- .../common/status/MonitorableJob.java | 7 +- .../datatools/manager/DataManager.java | 3 + .../controllers/api/DeploymentController.java | 2 +- .../controllers/api/ServerController.java | 135 +++++++ .../datatools/manager/jobs/DeployJob.java | 331 +++++++++++++++--- .../manager/jobs/MonitorServerStatusJob.java | 147 ++++++++ .../datatools/manager/models/Deployment.java | 23 +- .../datatools/manager/models/OtpServer.java | 35 +- .../datatools/manager/models/Project.java | 23 +- .../manager/persistence/Persistence.java | 7 +- 10 files changed, 626 insertions(+), 87 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java diff --git a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java index 7853beea5..ea904da90 100644 --- a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java +++ b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java @@ -55,7 +55,8 @@ public enum JobType { EXPORT_SNAPSHOT_TO_GTFS, CONVERT_EDITOR_MAPDB_TO_SQL, VALIDATE_ALL_FEEDS, - MERGE_PROJECT_FEEDS + MERGE_PROJECT_FEEDS, + MONITOR_SERVER_STATUS } public MonitorableJob(String owner, String name, JobType type) { @@ -65,10 +66,6 @@ public MonitorableJob(String owner, String name, JobType type) { registerJob(); } - public MonitorableJob(String owner) { - this(owner, "Unnamed Job", JobType.UNKNOWN_TYPE); - } - /** * This method should never be called directly or overridden. * It is a standard start-up stage for all monitorable jobs. diff --git a/src/main/java/com/conveyal/datatools/manager/DataManager.java b/src/main/java/com/conveyal/datatools/manager/DataManager.java index 49abc3178..ef7cfb609 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataManager.java +++ b/src/main/java/com/conveyal/datatools/manager/DataManager.java @@ -16,6 +16,7 @@ import com.conveyal.datatools.manager.controllers.api.OrganizationController; import com.conveyal.datatools.manager.controllers.api.ProjectController; import com.conveyal.datatools.manager.controllers.api.AppInfoController; +import com.conveyal.datatools.manager.controllers.api.ServerController; import com.conveyal.datatools.manager.controllers.api.StatusController; import com.conveyal.datatools.manager.controllers.api.UserController; import com.conveyal.datatools.manager.extensions.ExternalFeedResource; @@ -213,6 +214,7 @@ static void registerRoutes() throws IOException { NoteController.register(API_PREFIX); StatusController.register(API_PREFIX); OrganizationController.register(API_PREFIX); + ServerController.register(API_PREFIX); // Register editor API routes if (isModuleEnabled("editor")) { @@ -224,6 +226,7 @@ static void registerRoutes() throws IOException { gtfsConfig = yamlMapper.readTree(gtfs); new EditorControllerImpl(EDITOR_API_PREFIX, Table.AGENCY, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.CALENDAR, DataManager.GTFS_DATA_SOURCE); + // NOTE: fare_attributes controller handles updates to nested table fare_rules. new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_ATTRIBUTES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index cc8c60e68..a4e873029 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -255,7 +255,7 @@ private static String deploy (Request req, Response res) { // no risk that these values can overlap. This may be over engineering this system though. The user deploying // a set of feeds would likely not create two deployment targets with the same name (and the name is unlikely // to change often). - OtpServer otpServer = project.retrieveServer(target); + OtpServer otpServer = Persistence.servers.getById(target); if (otpServer == null) haltWithMessage(req, 400, "Must provide valid OTP server target ID."); // Check that permissions of user allow them to deploy to target. boolean isProjectAdmin = userProfile.canAdministerProject(deployment.projectId, deployment.organizationId()); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java new file mode 100644 index 000000000..772414cfd --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -0,0 +1,135 @@ +package com.conveyal.datatools.manager.controllers.api; + +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; +import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException; +import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetGroupsRequest; +import com.amazonaws.services.elasticloadbalancingv2.model.TargetGroup; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.JsonViews; +import com.conveyal.datatools.manager.models.OtpServer; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.json.JsonManager; +import org.bson.Document; +import org.eclipse.jetty.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import spark.Request; +import spark.Response; + +import java.util.List; + +import static com.conveyal.datatools.common.utils.SparkUtils.haltWithMessage; +import static spark.Spark.delete; +import static spark.Spark.options; +import static spark.Spark.post; +import static spark.Spark.put; + +/** + * Handlers for HTTP API requests that affect deployment Servers. + * These methods are mapped to API endpoints by Spark. + */ +public class ServerController { + private static JsonManager json = new JsonManager<>(OtpServer.class, JsonViews.UserInterface.class); + private static final Logger LOG = LoggerFactory.getLogger(ServerController.class); + + /** + * Gets the server specified by the request's id parameter and ensure that user has access to the + * deployment. If the user does not have permission the Spark request is halted with an error. + */ + private static OtpServer checkServerPermissions(Request req, Response res) { + Auth0UserProfile userProfile = req.attribute("user"); + String serverId = req.params("id"); + OtpServer server = Persistence.servers.getById(serverId); + if (server == null) { + haltWithMessage(req, HttpStatus.BAD_REQUEST_400, "Server does not exist."); + } + boolean isProjectAdmin = userProfile.canAdministerProject(server.projectId, server.organizationId()); + if (!isProjectAdmin && !userProfile.getUser_id().equals(server.user())) { + // If user is not a project admin and did not create the deployment, access to the deployment is denied. + haltWithMessage(req, HttpStatus.UNAUTHORIZED_401, "User not authorized for deployment."); + } + return server; + } + + private static OtpServer deleteServer(Request req, Response res) { + OtpServer server = checkServerPermissions(req, res); + server.delete(); + return server; + } + + /** + * Create a new server for the project. All feed sources with a valid latest version are added to the new + * deployment. + */ + private static OtpServer createServer(Request req, Response res) { + // TODO error handling when request is bogus + // TODO factor out user profile fetching, permissions checks etc. + Auth0UserProfile userProfile = req.attribute("user"); + Document newServerFields = Document.parse(req.body()); + String projectId = newServerFields.getString("projectId"); + String organizationId = newServerFields.getString("organizationId"); + if (projectId == null) haltWithMessage(req, 400, "Must provide valid project ID"); + boolean allowedToCreate = userProfile.canAdministerProject(projectId, organizationId); + + if (allowedToCreate) { + Project project = Persistence.projects.getById(projectId); + OtpServer newServer = new OtpServer(); + validateFields(req, newServerFields); + // FIXME: Here we are creating a deployment and updating it with the JSON string (two db operations) + // We do this because there is not currently apply JSON directly to an object (outside of Mongo codec + // operations) + Persistence.servers.create(newServer); + return Persistence.servers.update(newServer.id, req.body()); + } else { + haltWithMessage(req, 403, "Not authorized to create a server for project " + projectId); + return null; + } + } + + /** + * Update a single server. If the server's feed versions are updated, checks to ensure that each + * version exists and is a part of the same parent project are performed before updating. + */ + private static OtpServer updateServer(Request req, Response res) { + OtpServer serverToUpdate = checkServerPermissions(req, res); + Document updateDocument = Document.parse(req.body()); + Auth0UserProfile user = req.attribute("user"); + if (serverToUpdate.admin && !user.canAdministerApplication()) { + haltWithMessage(req, 401, "User cannot modify admin-only server."); + } + // FIXME use generic update hook, also feedVersions is getting serialized into MongoDB (which is undesirable) + validateFields(req, updateDocument); + OtpServer updatedServer = Persistence.servers.update(serverToUpdate.id, req.body()); + return updatedServer; + } + + private static void validateFields(Request req, Document document) { + if (document.containsKey("targetGroupArn") && document.get("targetGroupArn") != null) { + // Validate that the Target Group ARN is valid. + try { + DescribeTargetGroupsRequest describeTargetGroupsRequest = new DescribeTargetGroupsRequest().withTargetGroupArns(document.get("targetGroupArn").toString()); + AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); + List targetGroups = elb.describeTargetGroups(describeTargetGroupsRequest).getTargetGroups(); + if (targetGroups.size() == 0) { + haltWithMessage(req, 400, "Invalid value for Target Group ARN. Could not locate Target Group."); + } + } catch (AmazonElasticLoadBalancingException e) { + haltWithMessage(req, 400, "Invalid value for Target Group ARN."); + } + + } + } + + /** + * Register HTTP methods with handler methods. NOTE: there is no GET server endpoint because servers are fetched as + * nested entities under Projects. + */ + public static void register (String apiPrefix) { + options(apiPrefix + "secure/servers", (q, s) -> ""); + delete(apiPrefix + "secure/servers/:id", ServerController::deleteServer, json::write); + post(apiPrefix + "secure/servers", ServerController::createServer, json::write); + put(apiPrefix + "secure/servers/:id", ServerController::updateServer, json::write); + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 3f143f9d7..021e32b97 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -4,11 +4,21 @@ import com.amazonaws.event.ProgressListener; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.AmazonEC2Exception; import com.amazonaws.services.ec2.model.CreateTagsRequest; +import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; +import com.amazonaws.services.ec2.model.IamInstanceProfileSpecification; import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.InstanceNetworkInterfaceSpecification; +import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.ec2.model.RunInstancesRequest; -import com.amazonaws.services.ec2.model.RunInstancesResult; import com.amazonaws.services.ec2.model.Tag; +import com.amazonaws.services.ec2.model.TerminateInstancesRequest; +import com.amazonaws.services.ec2.model.TerminateInstancesResult; +import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest; +import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsResult; +import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @@ -19,17 +29,27 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.io.Serializable; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Scanner; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import com.amazonaws.waiters.Waiter; +import com.amazonaws.waiters.WaiterParameters; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.models.Deployment; @@ -41,11 +61,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.mongodb.client.model.Filters.and; -import static com.mongodb.client.model.Filters.eq; -import static com.mongodb.client.model.Filters.not; -import static com.mongodb.client.model.Updates.pull; -import static com.mongodb.client.model.Updates.set; +import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_OTP_VERSION; +import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_R5_VERSION; /** * Deploy the given deployment to the OTP servers specified by targets. @@ -55,7 +72,10 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); - private static final String bundlePrefix = "bundles/"; + private static final String bundlePrefix = "bundles"; + + private AmazonEC2 ec2; + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); /** The deployment to deploy */ private Deployment deployment; @@ -69,6 +89,12 @@ public class DeployJob extends MonitorableJob { /** This hides the status field on the parent class, providing additional fields. */ public DeployStatus status; + private int serverCounter = 0; +// private String imageId; + private String dateString = DATE_FORMAT.format(new Date()); + List amiNameFilter = Collections.singletonList("ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-????????"); + List amiStateFilter = Collections.singletonList("available"); + @JsonProperty public String getDeploymentId () { return deployment.id; @@ -85,11 +111,18 @@ public DeployJob(Deployment deployment, String owner, OtpServer otpServer) { status.built = false; status.numServersCompleted = 0; status.totalServers = otpServer.internalUrl == null ? 0 : otpServer.internalUrl.size(); + // CONNECT TO EC2 + // FIXME Should this ec2 client be longlived? + ec2 = AmazonEC2Client.builder().build(); +// imageId = ec2.describeImages(new DescribeImagesRequest().withOwners("099720109477").withFilters(new Filter("name", amiNameFilter), new Filter("state", amiStateFilter))).getImages().get(0).getImageId(); } public void jobLogic () { int targetCount = otpServer.internalUrl != null ? otpServer.internalUrl.size() : 0; int totalTasks = 1 + targetCount; + if (otpServer.s3Bucket != null) totalTasks++; + // FIXME + if (otpServer.targetGroupArn != null) totalTasks++; int tasksCompleted = 0; String statusMessage; @@ -107,7 +140,7 @@ public void jobLogic () { // Dump the deployment bundle to the temp file. try { - status.message = "Creating OTP Bundle"; + status.message = "Creating transit bundle (GTFS and OSM)"; this.deployment.dump(deploymentTempFile, true, true, true); tasksCompleted++; } catch (Exception e) { @@ -126,11 +159,11 @@ public void jobLogic () { if(otpServer.s3Bucket != null) { status.message = "Uploading to S3"; status.uploadingS3 = true; - LOG.info("Uploading deployment {} to s3", deployment.name); - String key = null; + String key = getS3BundleKey(); + LOG.info("Uploading deployment {} to s3://{}/{}", deployment.name, otpServer.s3Bucket, key); try { +// PutObjectRequest putObjectRequest = new PutObjectRequest(otpServer.s3Bucket, key, deploymentTempFile).; TransferManager tx = TransferManagerBuilder.standard().withS3Client(FeedStore.s3Client).build(); - key = bundlePrefix + deployment.parentProject().id + "/" + deployment.name + ".zip"; final Upload upload = tx.upload(otpServer.s3Bucket, key, deploymentTempFile); upload.addProgressListener((ProgressListener) progressEvent -> { @@ -145,10 +178,11 @@ public void jobLogic () { tx.shutdownNow(false); // copy to [name]-latest.zip - String copyKey = bundlePrefix + deployment.parentProject().id + "/" + deployment.parentProject().name.toLowerCase() + "-latest.zip"; + String copyKey = getLatestS3BundleKey(); CopyObjectRequest copyObjRequest = new CopyObjectRequest( otpServer.s3Bucket, key, otpServer.s3Bucket, copyKey); FeedStore.s3Client.copyObject(copyObjRequest); + LOG.info("Copied to s3://{}/{}", otpServer.s3Bucket, copyKey); } catch (AmazonClientException|InterruptedException e) { statusMessage = String.format("Error uploading (or copying) deployment bundle to s3://%s/%s", otpServer.s3Bucket, key); LOG.error(statusMessage); @@ -156,11 +190,12 @@ public void jobLogic () { status.fail(statusMessage); return; } - + LOG.info("Uploaded to s3://{}/{}", otpServer.s3Bucket, getS3BundleKey()); + status.update("Upload to S3 complete.", status.percentComplete + 10); status.uploadingS3 = false; } - if (otpServer.createServer) { + if (otpServer.targetGroupArn != null) { if ("true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled"))) { createServer(); // If creating a new server, there is no need to deploy to an existing one. @@ -320,6 +355,14 @@ public void jobLogic () { status.baseUrl = otpServer.publicUrl; } + private String getS3BundleKey() { + return String.format("%s/%s/%s.zip", bundlePrefix, deployment.projectId, this.jobId); + } + + private String getLatestS3BundleKey() { + return String.format("%s/%s/%s-latest.zip", bundlePrefix, deployment.projectId, deployment.parentProject().name.toLowerCase()); + } + @Override public void jobFinished () { // Delete temp file containing OTP deployment (OSM extract and GTFS files) so that the server's disk storage @@ -333,8 +376,8 @@ public void jobFinished () { // Update status with successful completion state only if no error was encountered. status.update(false, "Deployment complete!", 100, true); // Store the target server in the deployedTo field. - LOG.info("Updating deployment target to {} id={}", otpServer.target(), deployment.id); - Persistence.deployments.updateField(deployment.id, "deployedTo", otpServer.target()); + LOG.info("Updating deployment target to {} id={}", otpServer.id, deployment.id); + Persistence.deployments.updateField(deployment.id, "deployedTo", otpServer.id); // Update last deployed field. Persistence.deployments.updateField(deployment.id, "lastDeployed", new Date()); message = String.format("Deployment %s successfully deployed to %s", deployment.name, otpServer.publicUrl); @@ -347,45 +390,235 @@ public void jobFinished () { public void createServer () { try { - // CONNECT TO EC2 - AmazonEC2 ec2 = new AmazonEC2Client(); - ec2.setEndpoint("ec2.us-east-1.amazonaws.com"); - // User data should contain info about: - // 1. Downloading GTFS/OSM info (s3) - // 2. Time to live until shutdown/termination (for test servers) - // 3. Hosting / nginx - String myUserData = "hello"; - // CREATE EC2 INSTANCES - RunInstancesRequest runInstancesRequest = new RunInstancesRequest() - .withInstanceType("t1.micro") - .withMinCount(1) - .withSubnetId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.subnet")) - .withImageId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ami")) - .withKeyName(DataManager.getConfigPropertyAsText("modules.deployment.ec2.keyName")) - .withInstanceInitiatedShutdownBehavior("terminate") - .withMaxCount(1) - .withSecurityGroupIds(DataManager.getConfigPropertyAsText("modules.deployment.ec2.securityGroup")) - .withUserData(Base64.encodeBase64String(myUserData.getBytes())); - - RunInstancesResult runInstances = ec2.runInstances(runInstancesRequest); - - // TAG EC2 INSTANCES - List instances = runInstances.getReservation().getInstances(); - int idx = 1; - for (Instance instance : instances) { - CreateTagsRequest createTagsRequest = new CreateTagsRequest(); - createTagsRequest.withResources(instance.getInstanceId()) // - .withTags(new Tag("Name", "otp-" + idx)); - ec2.createTags(createTagsRequest); - - idx++; + // First start graph-building instance and wait for graph to successfully build. + List instances = startEC2Instances(1); + if (instances.size() > 1) { + // FIXME is this check/shutdown entirely unnecessary? + status.fail("CRITICAL: More than one server initialized for graph building. Cancelling job. Please contact system administrator."); + // Terminate new instances. + // FIXME Should this ec2 client be longlived? + ec2.terminateInstances(new TerminateInstancesRequest(getIds(instances))); } + // FIXME What if instances list is empty? + MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob(owner, deployment, instances.get(0), otpServer); + monitorInitialServerJob.run(); + status.update("Graph build is complete!", 50); + // Spin up remaining servers which will download the graph from S3. + int remainingServerCount = otpServer.instanceCount <= 0 ? 0 : otpServer.instanceCount - 1; + if (remainingServerCount > 0) { + // Spin up remaining EC2 instances. + List remainingInstances = startEC2Instances(remainingServerCount); + instances.addAll(remainingInstances); + // Create new thread pool to monitor server setup so that the servers are monitored in parallel. + ExecutorService service = Executors.newFixedThreadPool(remainingServerCount); + for (Instance instance : remainingInstances) { + // Note: new instances are added + MonitorServerStatusJob monitorServerStatusJob = new MonitorServerStatusJob(owner, deployment, instance, otpServer); + service.submit(monitorServerStatusJob); + } + // Shutdown thread pool once the jobs are completed and wait for its termination. Once terminated, we can + // consider the servers up and running (or they have failed to initialize properly). + service.shutdown(); + service.awaitTermination(4, TimeUnit.HOURS); + } + String finalMessage = "Server setup is complete!"; + if (otpServer.instanceIds != null) { + // Deregister old instances from load balancer. (Note: new instances are registered with load balancer in + // MonitorServerStatusJob.) + LOG.info("Deregistering instances from load balancer {}", otpServer.instanceIds); + TargetDescription[] targetDescriptions = otpServer.instanceIds + .stream() + .map(id -> new TargetDescription().withId(id)).toArray(TargetDescription[]::new); + DeregisterTargetsRequest deregisterTargetsRequest = new DeregisterTargetsRequest() + .withTargetGroupArn(otpServer.targetGroupArn) + .withTargets(targetDescriptions); + DeregisterTargetsResult deregisterTargetsResult = com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient.builder().build() + .deregisterTargets(deregisterTargetsRequest); + // Terminate old instances. + LOG.info("Terminating instances {}", otpServer.instanceIds); + try { + TerminateInstancesRequest terminateInstancesRequest = new TerminateInstancesRequest().withInstanceIds(otpServer.instanceIds); + TerminateInstancesResult terminateInstancesResult = ec2.terminateInstances(terminateInstancesRequest); + } catch (AmazonEC2Exception e) { + LOG.warn("Could not terminate EC2 instances {}", otpServer.instanceIds); + finalMessage = String.format("Server setup is complete! (WARNING: Could not terminate previous EC2 instances: %s", otpServer.instanceIds); + } + } + // Update list of instance IDs with new list. + Persistence.servers.updateField(otpServer.id, "instanceIds", getIds(instances)); + // Job is complete? FIXME Do we need a status check here? + status.update(false, finalMessage, 100, true); } catch (Exception e) { LOG.error("Could not deploy to EC2 server", e); status.fail("Could not deploy to EC2 server", e); } } + private List startEC2Instances(int count) { + // User data should contain info about: + // 1. Downloading GTFS/OSM info (s3) + // 2. Time to live until shutdown/termination (for test servers) + // 3. Hosting / nginx + // FIXME: Allow for r5 servers to be created. + String userData = constructUserData(otpServer.s3Bucket, deployment.r5); + // The subnet ID should only change if starting up a server in some other AWS account. This is not + // likely to be a requirement. + // Define network interface so that a public IP can be associated with server. + InstanceNetworkInterfaceSpecification interfaceSpecification = new InstanceNetworkInterfaceSpecification() + .withSubnetId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.subnet")) + .withAssociatePublicIpAddress(true) + .withGroups(DataManager.getConfigPropertyAsText("modules.deployment.ec2.securityGroup")) + .withDeviceIndex(0); + + RunInstancesRequest runInstancesRequest = new RunInstancesRequest() + .withNetworkInterfaces(interfaceSpecification) + .withInstanceType(otpServer.instanceType) + .withMinCount(count) + .withMaxCount(count) + .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(DataManager.getConfigPropertyAsText("modules.deployment.ec2.arn"))) + .withImageId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ami")) + .withKeyName(DataManager.getConfigPropertyAsText("modules.deployment.ec2.keyName")) + // This will have the instance terminate when it is shut down. + .withInstanceInitiatedShutdownBehavior("terminate") + .withUserData(Base64.encodeBase64String(userData.getBytes())); + final List instances = ec2.runInstances(runInstancesRequest).getReservation().getInstances(); + + List instanceIds = getIds(instances); + Map instanceIpAddresses = new HashMap<>(); + // Wait so that create tags request does not fail because instances not found. + try { + Waiter waiter = ec2.waiters().instanceStatusOk(); +// ec2.waiters().systemStatusOk() + long beginWaiting = System.currentTimeMillis(); + waiter.run(new WaiterParameters<>(new DescribeInstanceStatusRequest().withInstanceIds(instanceIds))); + LOG.info("Instance status is OK after {} ms", (System.currentTimeMillis() - beginWaiting)); + } catch (Exception e) { + LOG.error("Waiter for instance status check failed.", e); + status.fail("Waiter for instance status check failed."); + // FIXME: Terminate instance??? + return Collections.EMPTY_LIST; + } + for (Instance instance : instances) { + // The public IP addresses will likely be null at this point because they take a few seconds to initialize. + instanceIpAddresses.put(instance.getInstanceId(), instance.getPublicIpAddress()); + String serverName = String.format("%s %s (%s) %d", deployment.r5 ? "r5" : "otp", deployment.name, dateString, serverCounter++); + LOG.info("Creating tags for new EC2 instance {}", serverName); + ec2.createTags(new CreateTagsRequest() + .withTags(new Tag("Name", serverName)) + .withTags(new Tag("projectId", deployment.projectId)) + .withResources(instance.getInstanceId()) + ); + } + List updatedInstances = new ArrayList<>(); + while (instanceIpAddresses.values().contains(null)) { + LOG.info("Checking that public IP addresses have initialized for EC2 instances."); + // Reset instances list so that updated instances have the latest state information (e.g., public IP has + // been assigned). + updatedInstances.clear(); + // Check that all of the instances have public IPs. + DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest().withInstanceIds(instanceIds); + List reservations = ec2.describeInstances(describeInstancesRequest).getReservations(); + for (Reservation reservation : reservations) { + for (Instance instance : reservation.getInstances()) { + instanceIpAddresses.put(instance.getInstanceId(), instance.getPublicIpAddress()); + updatedInstances.add(instance); + } + } + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + LOG.info("Public IP addresses have all been assigned. {}", instanceIpAddresses.values().toString()); + return updatedInstances; + } + + private List getIds (List instances) { + return instances.stream().map(Instance::getInstanceId).collect(Collectors.toList()); + } + + private String constructUserData(String s3Bucket, boolean r5) { + // Prefix/name of JAR file (WITHOUT .jar) FIXME: make this configurable. + String jarName = r5 ? deployment.r5Version : deployment.otpVersion; + if (jarName == null) { + // If there is no version specified, use the default (and persist value). + jarName = r5 ? DEFAULT_R5_VERSION : DEFAULT_OTP_VERSION; + Persistence.deployments.updateField(deployment.id, r5 ? "r5Version" : "otpVersion", jarName); + } + String tripPlanner = r5 ? "r5" : "otp"; + String s3JarBucket = r5 ? "r5-builds" : "opentripplanner-builds"; + String s3JarUrl = String.format("s3://%s/%s.jar", s3JarBucket, jarName); + // FIXME Check that jar URL exists. + String jarDir = String.format("/opt/%s", tripPlanner); + String s3BundlePath = String.format("s3://%s/%s", s3Bucket, getS3BundleKey()); + boolean graphAlreadyBuilt = FeedStore.s3Client.doesObjectExist(s3Bucket, getS3GraphKey()); + List lines = new ArrayList<>(); + String routerName = "default"; + String routerDir = String.format("/var/%s/graphs/%s", tripPlanner, routerName); + // BEGIN USER DATA + lines.add("#!/bin/bash"); + // Send trip planner logs to LOGFILE + lines.add(String.format("BUILDLOGFILE=/var/log/%s-build.log", tripPlanner)); + lines.add(String.format("LOGFILE=/var/log/%s.log", tripPlanner)); + // Log user data setup to /var/log/user-data.log + lines.add("exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1"); + // Install the necessary files to run the trip planner. FIXME Remove. This should be captured by the AMI. +// lines.add("apt update -y"); +// lines.add("apt install -y openjdk-8-jre"); +// lines.add("apt install -y openjfx"); +// lines.add("apt install -y awscli"); +// lines.add("apt install -y unzip"); + // Create the directory for the graph inputs. + lines.add(String.format("mkdir -p %s", routerDir)); + lines.add(String.format("chown ubuntu %s", routerDir)); + // Remove the current inputs and replace with inputs from S3. + lines.add(String.format("rm -rf %s/*", routerDir)); + lines.add(String.format("aws s3 --region us-east-1 cp %s /tmp/bundle.zip", s3BundlePath)); + lines.add(String.format("unzip /tmp/bundle.zip -d %s", routerDir)); + // FIXME: Add ability to fetch custom bikeshare.xml file (CarFreeAtoZ) + if (false) { + lines.add(String.format("wget -O %s/bikeshare.xml ${config.bikeshareFeed}", routerDir)); + lines.add(String.format("printf \"{\\n bikeRentalFile: \"bikeshare.xml\"\\n}\" >> %s/build-config.json\"", routerDir)); + } + // Download trip planner JAR. + lines.add(String.format("mkdir -p %s", jarDir)); + String region = deployment.r5 ? "eu-west-1" : "us-east-1"; + lines.add(String.format("aws s3 cp --region %s %s %s/%s.jar", region, s3JarUrl, jarDir, jarName)); + // Kill any running java process FIXME Currently the AMI we're using starts up OTP on startup, so we need to kill the java process in order to follow the below instructions + lines.add("sudo pkill -9 -f java"); + if (graphAlreadyBuilt) { + lines.add("echo 'downloading graph from s3'"); + // Download Graph from S3 and spin up trip planner. + lines.add(String.format("aws s3 --region us-east-1 cp %s %s/Graph.obj", getS3GraphPath(), routerDir)); + } else { + lines.add("echo 'starting graph build'"); + // Build the graph if Graph object (presumably this is the first instance to be started up). + if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu java -Xmx6G -jar %s/%s.jar point --build %s", jarDir, jarName, routerDir)); + else lines.add(String.format("sudo -H -u ubuntu java -jar %s/%s.jar --build %s > $BUILDLOGFILE 2>&1", jarDir, jarName, routerDir)); + // Upload the graph to S3. + if (!deployment.r5) lines.add(String.format("aws s3 --region us-east-1 cp %s/Graph.obj %s", routerDir, getS3GraphPath())); + } + if (deployment.buildGraphOnly) { + lines.add("echo 'shutting down server (build graph only specified in deployment target)'"); + lines.add("sudo poweroff"); + } else { + lines.add("echo 'kicking off trip planner (logs at $LOGFILE)'"); + // Kick off the application. FIXME use sudo service opentripplanner start? + if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu nohup java -Xmx6G -Djava.util.Arrays.useLegacyMergeSort=true -jar %s/%s.jar point --isochrones %s > /var/log/r5.out 2>&1&", jarDir, jarName, routerDir)); + else lines.add(String.format("sudo -H -u ubuntu nohup java -jar %s/%s.jar --server --bindAddress 127.0.0.1 --router default > $LOGFILE 2>&1 &", jarDir, jarName)); + } + return String.join("\n", lines); + } + + private String getS3GraphKey() { + return String.format("%s/%s/Graph.obj", deployment.projectId, this.jobId); + } + + private String getS3GraphPath() { + return String.format("s3://%s/%s", otpServer.s3Bucket, getS3GraphKey()); + } + /** * Represents the current status of this job. */ diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java new file mode 100644 index 000000000..0928c4c49 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -0,0 +1,147 @@ +package com.conveyal.datatools.manager.jobs; + +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.InstanceStateChange; +import com.amazonaws.services.ec2.model.TerminateInstancesRequest; +import com.amazonaws.services.ec2.model.TerminateInstancesResult; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; +import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest; +import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsResult; +import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.models.Deployment; +import com.conveyal.datatools.manager.models.OtpServer; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class MonitorServerStatusJob extends MonitorableJob { + private static final Logger LOG = LoggerFactory.getLogger(MonitorServerStatusJob.class); + private final Deployment deployment; + private final Instance instance; + private final OtpServer otpServer; + private final AmazonEC2 ec2 = AmazonEC2Client.builder().build(); + private final CloseableHttpClient httpClient = HttpClients.createDefault(); + // If the job takes longer than XX seconds, fail the job. + private static final int TIMEOUT_MILLIS = 60 * 60 * 1000; // One hour + + public MonitorServerStatusJob(String owner, Deployment deployment, Instance instance, OtpServer otpServer) { + super( + owner, + String.format("Monitor server setup %s", instance.getPublicIpAddress()), + JobType.MONITOR_SERVER_STATUS + ); + this.deployment = deployment; + this.instance = instance; + this.otpServer = otpServer; + status.message = "Checking server status..."; + } + + @Override + public void jobLogic() { + long startTime = System.currentTimeMillis(); + // FIXME use private IP? + String otpUrl = String.format("http://%s/otp", instance.getPublicIpAddress()); + boolean otpIsRunning = false, routerIsAvailable = false; + // Progressively check status of OTP server + if (deployment.buildGraphOnly) { + // FIXME No need to check that OTP is running. Just check to see that the graph is built. +// FeedStore.s3Client.doesObjectExist(otpServer.s3Bucket, ); + } + // First, check that OTP has started up. + status.update("Instance status is OK.", 20); + while (!otpIsRunning) { + LOG.info("Checking that OTP is running on server at {}.", otpUrl); + // If the request is successful, the OTP instance has started. + otpIsRunning = checkForSuccessfulRequest(otpUrl, 5); + if (System.currentTimeMillis() - startTime > TIMEOUT_MILLIS) { + status.fail(String.format("Job timed out while monitoring setup for server %s", instance.getInstanceId())); + return; + } + } + String otpMessage = String.format("OTP is running on server %s (%s). Building graph...", instance.getInstanceId(), otpUrl); + LOG.info(otpMessage); + status.update(otpMessage, 30); + // Once this is confirmed, check for the existence of the router, which will indicate that the graph build is + // complete. + String routerUrl = String.format("%s/routers/default", otpUrl); + while (!routerIsAvailable) { + LOG.info("Checking that router is available (i.e., graph build or read is finished) at {}", routerUrl); + // If the request was successful, the graph build is complete! + // TODO: Substitute in specific router ID? Or just default to... default. + routerIsAvailable = checkForSuccessfulRequest(routerUrl, 5); + } + status.update(String.format("Graph build completed on server %s (%s).", instance.getInstanceId(), routerUrl), 90); + if (otpServer.targetGroupArn != null) { + // After the router is available, the EC2 instance can be registered with the load balancer. + // REGISTER INSTANCE WITH LOAD BALANCER + AmazonElasticLoadBalancing elbClient = AmazonElasticLoadBalancingClient.builder().build(); + RegisterTargetsRequest registerTargetsRequest = new RegisterTargetsRequest() + .withTargetGroupArn(otpServer.targetGroupArn) + .withTargets(new TargetDescription().withId(instance.getInstanceId())); + RegisterTargetsResult registerTargetsResult = elbClient.registerTargets(registerTargetsRequest); +// try { +// elbClient.waiters().targetInService().run(new WaiterParameters<>(new DescribeTargetHealthRequest().withTargetGroupArn(otpServer.targetGroupArn))); +// } catch (Exception e) { +// e.printStackTrace(); +// } + // FIXME how do we know it was successful? + String message = String.format("Server %s successfully registered with load balancer %s", instance.getInstanceId(), otpServer.targetGroupArn); + LOG.info(message); + status.update(false, message, 100, true); + } else { + LOG.info("There is no load balancer under which to register ec2 instance {}.", instance.getInstanceId()); + } + } + + /** + * Checks the provided URL for a successful response (i.e., HTTP status code is 200). + */ + private boolean checkForSuccessfulRequest(String url, int delaySeconds) { + // Wait for the specified seconds before the request is initiated. + try { + Thread.sleep(1000 * delaySeconds); + } catch (InterruptedException e) { + e.printStackTrace(); + } + HttpGet httpGet = new HttpGet(url); + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + HttpEntity entity = response.getEntity(); + int statusCode = response.getStatusLine().getStatusCode(); + // Ensure the response body is fully consumed + EntityUtils.consume(entity); + return statusCode == 200; + } catch (IOException e) { + LOG.error("Could not complete request to {}", url); + e.printStackTrace(); + } + return false; + } + + @Override + public void jobFinished() { + if (status.error) { + // Terminate server. + TerminateInstancesResult terminateInstancesResult = ec2.terminateInstances( + new TerminateInstancesRequest().withInstanceIds(instance.getInstanceId()) + ); + InstanceStateChange instanceStateChange = terminateInstancesResult.getTerminatingInstances().get(0); + // If instance state code is 48 that means it has been terminated. + if (instanceStateChange.getCurrentState().getCode() == 48) { + // FIXME: this message will not make it to the client because the status has already been failed. Also, + // I'm not sure if this is even the right way to handle the instance state check. + status.update("Instance is terminated!", 100); + } + } + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 42ed33bc4..15caebb5d 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -23,7 +23,6 @@ import java.io.InputStream; import java.io.Serializable; import java.net.HttpURLConnection; -import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.text.DateFormat; @@ -42,7 +41,6 @@ import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; -import static com.mongodb.client.model.Filters.not; /** * A deployment of (a given version of) OTP on a given set of feeds. @@ -57,6 +55,9 @@ public class Deployment extends Model implements Serializable { public String name; + public static final String DEFAULT_OTP_VERSION = "otp-v1.3.0"; + public static final String DEFAULT_R5_VERSION = "v2.4.1-9-g3be6daa"; + /** What server is this currently deployed to? */ public String deployedTo; @@ -118,8 +119,22 @@ public void storeFeedVersions(Collection versions) { // future use public String osmFileId; - /** The commit of OTP being used on this deployment */ - public String otpCommit; + /** + * The version (according to git describe) of OTP being used on this deployment This should default to + * {@link Deployment#DEFAULT_OTP_VERSION}. + */ + public String otpVersion; + + public boolean buildGraphOnly; + + /** + * The version (according to git describe) of R5 being used on this deployment. This should default to + * {@link Deployment#DEFAULT_R5_VERSION}. + */ + public String r5Version; + + /** Whether this deployment should build an r5 server (false=OTP) */ + public boolean r5; /** Date when the deployment was last deployed to a server */ public Date lastDeployed; diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java index c60ada3a5..eef5c6693 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java @@ -1,28 +1,47 @@ package com.conveyal.datatools.manager.models; -import java.io.Serializable; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.List; /** * Created by landon on 5/20/16. */ -public class OtpServer implements Serializable { +public class OtpServer extends Model { private static final long serialVersionUID = 1L; public String name; public List internalUrl; + public List instanceIds; + public String instanceType; + public int instanceCount; + public String projectId; + public String targetGroupArn; public String publicUrl; - public Boolean admin; + public boolean admin; public String s3Bucket; public String s3Credentials; public boolean createServer; + /** Empty constructor for serialization. */ + public OtpServer () {} + + @JsonProperty("organizationId") + public String organizationId() { + Project project = parentProject(); + return project == null ? null : project.organizationId; + } + + public Project parentProject() { + return Persistence.projects.getById(projectId); + } + /** - * Convert the name field into a string with no special characters. + * Nothing fancy here. Just delete the Mongo record. * - * FIXME: This is currently used to keep track of which deployments have been deployed to which servers (it is used - * for the {@link Deployment#deployedTo} field), but we should likely. + * TODO should this also check refs in deployments? */ - public String target() { - return name != null ? name.replaceAll("[^a-zA-Z0-9]", "_") : null; + public void delete () { + Persistence.servers.removeById(this.id); } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 4757b0e35..1ee87c9cc 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,23 +35,11 @@ public class Project extends Model { public OtpRouterConfig routerConfig; - public Collection otpServers; - public String organizationId; - /** - * Locate and return an OTP server contained within the project that matches the name argument. - */ - public OtpServer retrieveServer(String name) { - if (name == null) return null; - for (OtpServer otpServer : otpServers) { - if (otpServer.name == null) continue; - if (name.equals(otpServer.name) || name.equals(otpServer.target())) { - return otpServer; - } - } - LOG.warn("Could not find OTP server with name {}", name); - return null; + @JsonProperty + public List getOtpServers () { + return Persistence.servers.getFiltered(eq("projectId", this.id)); } public String defaultTimeZone; @@ -86,9 +75,7 @@ public Collection retrieveProjectFeedSources() { * Get all the deployments for this project. */ public Collection retrieveDeployments() { - List deployments = Persistence.deployments - .getFiltered(eq("projectId", this.id)); - return deployments; + return Persistence.deployments.getFiltered(eq("projectId", this.id)); } // TODO: Does this need to be returned with JSON API response diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java index 9359340d3..d7d497671 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java @@ -11,6 +11,7 @@ import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Note; import com.conveyal.datatools.manager.models.Organization; +import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.models.Snapshot; import com.mongodb.MongoClient; @@ -47,8 +48,9 @@ public class Persistence { public static TypedPersistence notes; public static TypedPersistence organizations; public static TypedPersistence externalFeedSourceProperties; - public static TypedPersistence tokens; + public static TypedPersistence servers; public static TypedPersistence snapshots; + public static TypedPersistence tokens; public static void initialize () { @@ -90,8 +92,9 @@ public static void initialize () { notes = new TypedPersistence(mongoDatabase, Note.class); organizations = new TypedPersistence(mongoDatabase, Organization.class); externalFeedSourceProperties = new TypedPersistence(mongoDatabase, ExternalFeedSourceProperty.class); - tokens = new TypedPersistence(mongoDatabase, FeedDownloadToken.class); + servers = new TypedPersistence(mongoDatabase, OtpServer.class); snapshots = new TypedPersistence(mongoDatabase, Snapshot.class); + tokens = new TypedPersistence(mongoDatabase, FeedDownloadToken.class); // TODO: Set up indexes on feed versions by feedSourceId, version #? deployments, feedSources by projectId. // deployments.getMongoCollection().createIndex(Indexes.descending("projectId")); From f777b025ab2afddec6434669ffadc6a797b836ab Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 14:40:07 -0500 Subject: [PATCH 05/42] fix(user-mgmt): better error handling when Auth0 cannot update/create a user --- .../controllers/api/UserController.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/UserController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/UserController.java index eb6ff7793..106aa676a 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/UserController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/UserController.java @@ -22,6 +22,7 @@ import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; +import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; @@ -187,10 +188,20 @@ private static String createUser(Request req, Response res) throws IOException { String result = EntityUtils.toString(response.getEntity()); int statusCode = response.getStatusLine().getStatusCode(); - if(statusCode >= 300) haltWithMessage(req, statusCode, response.toString()); - - System.out.println(result); - + if(statusCode >= 300) { + // If Auth0 status shows an error, throw a halt with a reasonably intelligible message. + LOG.error("Auth0 error encountered. Could not create user: {}", response.toString()); + String errorMessage; + switch (statusCode) { + case HttpStatus.CONFLICT_409: + errorMessage = String.format("User already exists for email address %s.", jsonNode.get("email")); + break; + default: + errorMessage = String.format("Error while creating user: %s.", HttpStatus.getMessage(statusCode)); + break; + } + haltWithMessage(req, statusCode, errorMessage); + } return result; } From 889837633f27957114fa1e21f9c7e768e09ffe05 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 14:41:59 -0500 Subject: [PATCH 06/42] fix: move toGtfsDate from deleted class to FeedTx --- .../com/conveyal/datatools/editor/datastore/FeedTx.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/editor/datastore/FeedTx.java b/src/main/java/com/conveyal/datatools/editor/datastore/FeedTx.java index 04760ee2f..ddaea5764 100644 --- a/src/main/java/com/conveyal/datatools/editor/datastore/FeedTx.java +++ b/src/main/java/com/conveyal/datatools/editor/datastore/FeedTx.java @@ -32,8 +32,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static com.conveyal.datatools.editor.jobs.ProcessGtfsSnapshotExport.toGtfsDate; - /** a transaction in an agency database */ public class FeedTx extends DatabaseTx { private static final Logger LOG = LoggerFactory.getLogger(FeedTx.class); @@ -124,6 +122,10 @@ public FeedTx(DB tx, boolean buildSecondaryIndices) { // editedSinceSnapshot = tx.getAtomicBoolean("editedSinceSnapshot") == null ? tx.createAtomicBoolean("editedSinceSnapshot", false) : editedSinceSnapshot; } + private static int toGtfsDate (LocalDate date) { + return date.getYear() * 10000 + date.getMonthValue() * 100 + date.getDayOfMonth(); + } + public void commit () { try { // editedSinceSnapshot.set(true); From fa5ce83a61a3865c5dc027880a13879fc197d1a2 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 14:42:28 -0500 Subject: [PATCH 07/42] refactor: fix whitespace --- .../datatools/manager/jobs/NotifyUsersForSubscriptionJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/NotifyUsersForSubscriptionJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/NotifyUsersForSubscriptionJob.java index 35bc01eca..00b0b055e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/NotifyUsersForSubscriptionJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/NotifyUsersForSubscriptionJob.java @@ -37,7 +37,7 @@ private NotifyUsersForSubscriptionJob(String subscriptionType, String target, St /** * Convenience method to create and schedule a notification job to notify subscribed users. */ - public static void createNotification(String subscriptionType, String target, String message) { + public static void createNotification(String subscriptionType, String target, String message) { if (APPLICATION_URL == null || !(APPLICATION_URL.startsWith("https://") || APPLICATION_URL.startsWith("http://"))) { LOG.error("application.public_url (value={}) property must be set to a valid URL in order to send notifications to users.", APPLICATION_URL); return; From e0a1eb6552e14a3ef25b1a7949c48d37d0b1cd86 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 14:45:00 -0500 Subject: [PATCH 08/42] refactor: add missing aws pom entry --- pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pom.xml b/pom.xml index cb3affd28..26768998d 100644 --- a/pom.xml +++ b/pom.xml @@ -324,6 +324,17 @@ 2.0.0.0 + + + com.amazonaws + aws-java-sdk-ec2 + 1.11.410 + + + com.amazonaws + aws-java-sdk-elasticloadbalancingv2 + 1.11.410 + From 61c04d2561727a4562e1e5953f9292631a06d6de Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 12 Nov 2018 15:06:18 -0500 Subject: [PATCH 09/42] build(pom): update gtfs-lib dependency --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 26768998d..aa1d8c1ee 100644 --- a/pom.xml +++ b/pom.xml @@ -222,7 +222,7 @@ com.conveyal gtfs-lib - 3.4.0-SNAPSHOT + 4.0.1-SNAPSHOT From 7231a95658fe3bec7c6e3614b356a5a1a394ee37 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 15 Nov 2018 09:54:16 -0500 Subject: [PATCH 10/42] feature(server-mgmt): manage deployment servers at the application level This change allows servers to be assigned to a specific project or left open for any project in the application. --- .../controllers/api/ServerController.java | 60 +++++++++++++++---- .../datatools/manager/models/Project.java | 15 ++++- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 772414cfd..0ae5c8704 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -15,13 +15,16 @@ import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import spark.HaltException; import spark.Request; import spark.Response; +import java.util.Collections; import java.util.List; import static com.conveyal.datatools.common.utils.SparkUtils.haltWithMessage; import static spark.Spark.delete; +import static spark.Spark.get; import static spark.Spark.options; import static spark.Spark.post; import static spark.Spark.put; @@ -70,11 +73,13 @@ private static OtpServer createServer(Request req, Response res) { Document newServerFields = Document.parse(req.body()); String projectId = newServerFields.getString("projectId"); String organizationId = newServerFields.getString("organizationId"); - if (projectId == null) haltWithMessage(req, 400, "Must provide valid project ID"); - boolean allowedToCreate = userProfile.canAdministerProject(projectId, organizationId); + // If server has no project ID specified, user must be an application admin to create it. Otherwise, they must + // be a project admin. + boolean allowedToCreate = projectId == null + ? userProfile.canAdministerApplication() + : userProfile.canAdministerProject(projectId, organizationId); if (allowedToCreate) { - Project project = Persistence.projects.getById(projectId); OtpServer newServer = new OtpServer(); validateFields(req, newServerFields); // FIXME: Here we are creating a deployment and updating it with the JSON string (two db operations) @@ -88,6 +93,22 @@ private static OtpServer createServer(Request req, Response res) { } } + /** + * HTTP controller to fetch all servers or servers assigned to a particular project. This should only be used for the + * management of these servers. For checking servers that a project can deploy to, use {@link Project#availableOtpServers()}. + */ + private static List fetchServers (Request req, Response res) { + String projectId = req.queryParams("projectId"); + Auth0UserProfile userProfile = req.attribute("user"); + if (projectId != null) { + Project project = Persistence.projects.getById(projectId); + if (project == null) haltWithMessage(req, 400, "Must provide a valid project ID."); + else if (userProfile.canAdministerProject(projectId, null)) return project.availableOtpServers(); + } + else if (userProfile.canAdministerApplication()) return Persistence.servers.getAll(); + return Collections.emptyList(); + } + /** * Update a single server. If the server's feed versions are updated, checks to ensure that each * version exists and is a part of the same parent project are performed before updating. @@ -96,20 +117,34 @@ private static OtpServer updateServer(Request req, Response res) { OtpServer serverToUpdate = checkServerPermissions(req, res); Document updateDocument = Document.parse(req.body()); Auth0UserProfile user = req.attribute("user"); - if (serverToUpdate.admin && !user.canAdministerApplication()) { - haltWithMessage(req, 401, "User cannot modify admin-only server."); + if ((serverToUpdate.admin || serverToUpdate.projectId == null) && !user.canAdministerApplication()) { + haltWithMessage(req, 401, "User cannot modify admin-only or application-wide server."); } - // FIXME use generic update hook, also feedVersions is getting serialized into MongoDB (which is undesirable) validateFields(req, updateDocument); - OtpServer updatedServer = Persistence.servers.update(serverToUpdate.id, req.body()); + OtpServer updatedServer = Persistence.servers.update(serverToUpdate.id, updateDocument); return updatedServer; } - private static void validateFields(Request req, Document document) { - if (document.containsKey("targetGroupArn") && document.get("targetGroupArn") != null) { + /** + * Validate certain fields found in the document representing a server. This also currently modifies the document by + * removing problematic date fields. + */ + private static void validateFields(Request req, Document serverDocument) throws HaltException { + // FIXME: There is an issue with updating a MongoDB record with JSON serialized date fields because these come + // back as integers. MongoDB writes these values into the database fine, but when it comes to converting them + // into POJOs, it throws exceptions about expecting date types. For now, we can just remove them here, but this + // speaks to the fragility of this system currently. + serverDocument.remove("lastUpdated"); + serverDocument.remove("dateCreated"); + if (serverDocument.containsKey("projectId") && serverDocument.get("projectId") != null) { + Project project = Persistence.projects.getById(serverDocument.get("projectId").toString()); + if (project == null) haltWithMessage(req, 400, "Must specify valid project ID."); + } + if (serverDocument.containsKey("targetGroupArn") && serverDocument.get("targetGroupArn") != null) { // Validate that the Target Group ARN is valid. try { - DescribeTargetGroupsRequest describeTargetGroupsRequest = new DescribeTargetGroupsRequest().withTargetGroupArns(document.get("targetGroupArn").toString()); + DescribeTargetGroupsRequest describeTargetGroupsRequest = + new DescribeTargetGroupsRequest().withTargetGroupArns(serverDocument.get("targetGroupArn").toString()); AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); List targetGroups = elb.describeTargetGroups(describeTargetGroupsRequest).getTargetGroups(); if (targetGroups.size() == 0) { @@ -118,17 +153,16 @@ private static void validateFields(Request req, Document document) { } catch (AmazonElasticLoadBalancingException e) { haltWithMessage(req, 400, "Invalid value for Target Group ARN."); } - } } /** - * Register HTTP methods with handler methods. NOTE: there is no GET server endpoint because servers are fetched as - * nested entities under Projects. + * Register HTTP methods with handler methods. */ public static void register (String apiPrefix) { options(apiPrefix + "secure/servers", (q, s) -> ""); delete(apiPrefix + "secure/servers/:id", ServerController::deleteServer, json::write); + get(apiPrefix + "secure/servers", ServerController::fetchServers, json::write); post(apiPrefix + "secure/servers", ServerController::createServer, json::write); put(apiPrefix + "secure/servers/:id", ServerController::updateServer, json::write); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 1ee87c9cc..c6391f960 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -11,6 +11,7 @@ import java.util.stream.Collectors; import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.or; /** * Represents a collection of feed sources that can be made into a deployment. @@ -37,9 +38,17 @@ public class Project extends Model { public String organizationId; - @JsonProperty - public List getOtpServers () { - return Persistence.servers.getFiltered(eq("projectId", this.id)); + /** + * A list of servers that are available to deploy project feeds/OSM to. This includes servers assigned to this + * project as well as those that belong to no project. + * @return + */ + @JsonProperty("otpServers") + public List availableOtpServers() { + return Persistence.servers.getFiltered(or( + eq("projectId", this.id), + eq("projectId", null) + )); } public String defaultTimeZone; From b3723033670e3fb5d5d72f756ab76773b970b7cf Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 29 Nov 2018 15:52:20 -0500 Subject: [PATCH 11/42] refactor(server-job): attach just the project ID to the merge feeds job --- .../datatools/manager/jobs/MergeProjectFeedsJob.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeProjectFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeProjectFeedsJob.java index 4710f8567..5428d1351 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeProjectFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeProjectFeedsJob.java @@ -7,6 +7,7 @@ import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.FeedStore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import org.slf4j.Logger; @@ -39,7 +40,12 @@ public class MergeProjectFeedsJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(MergeProjectFeedsJob.class); - public final Project project; + private final Project project; + + @JsonProperty + public String getProjectId () { + return project.id; + } public MergeProjectFeedsJob(Project project, String owner) { super(owner, "Merging project feeds for " + project.name, JobType.MERGE_PROJECT_FEEDS); From aa0553bdb84127af8533313a691b351e4fa7757a Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 30 Nov 2018 14:23:56 -0500 Subject: [PATCH 12/42] refactor(deploy): shuffle deploy job code for clarity --- .../datatools/manager/jobs/DeployJob.java | 135 +++++++++--------- 1 file changed, 71 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 021e32b97..6017b1035 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -73,6 +73,14 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); private static final String bundlePrefix = "bundles"; + /** + * S3 bucket to upload deployment to. If not null, uses {@link OtpServer#s3Bucket}. Otherwise, defaults to + * {@link DataManager#feedBucket} + * */ + private final String s3Bucket; + private final int targetCount; + private int tasksCompleted = 0; + private int totalTasks; private AmazonEC2 ec2; private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); @@ -89,6 +97,7 @@ public class DeployJob extends MonitorableJob { /** This hides the status field on the parent class, providing additional fields. */ public DeployStatus status; + private String statusMessage; private int serverCounter = 0; // private String imageId; private String dateString = DATE_FORMAT.format(new Date()); @@ -105,8 +114,11 @@ public DeployJob(Deployment deployment, String owner, OtpServer otpServer) { super(owner, "Deploying " + deployment.name, JobType.DEPLOY_TO_OTP); this.deployment = deployment; this.otpServer = otpServer; + this.s3Bucket = otpServer.s3Bucket != null ? otpServer.s3Bucket : DataManager.feedBucket; // Use a special subclass of status here that has additional fields this.status = new DeployStatus(); + this.targetCount = otpServer.internalUrl != null ? otpServer.internalUrl.size() : 0; + this.totalTasks = 1 + targetCount; status.message = "Initializing..."; status.built = false; status.numServersCompleted = 0; @@ -118,14 +130,9 @@ public DeployJob(Deployment deployment, String owner, OtpServer otpServer) { } public void jobLogic () { - int targetCount = otpServer.internalUrl != null ? otpServer.internalUrl.size() : 0; - int totalTasks = 1 + targetCount; if (otpServer.s3Bucket != null) totalTasks++; // FIXME if (otpServer.targetGroupArn != null) totalTasks++; - int tasksCompleted = 0; - String statusMessage; - try { deploymentTempFile = File.createTempFile("deployment", ".zip"); } catch (IOException e) { @@ -155,49 +162,22 @@ public void jobLogic () { LOG.info("Deployment pctComplete = {}", status.percentComplete); status.built = true; - // Upload to S3, if applicable - if(otpServer.s3Bucket != null) { - status.message = "Uploading to S3"; - status.uploadingS3 = true; - String key = getS3BundleKey(); - LOG.info("Uploading deployment {} to s3://{}/{}", deployment.name, otpServer.s3Bucket, key); + // Upload to S3, if specifically required by the OTPServer or needed for servers in the target group to fetch. + if (otpServer.s3Bucket != null || otpServer.targetGroupArn != null) { try { -// PutObjectRequest putObjectRequest = new PutObjectRequest(otpServer.s3Bucket, key, deploymentTempFile).; - TransferManager tx = TransferManagerBuilder.standard().withS3Client(FeedStore.s3Client).build(); - final Upload upload = tx.upload(otpServer.s3Bucket, key, deploymentTempFile); - - upload.addProgressListener((ProgressListener) progressEvent -> { - status.percentUploaded = upload.getProgress().getPercentTransferred(); - }); - - upload.waitForCompletion(); - - // Shutdown the Transfer Manager, but don't shut down the underlying S3 client. - // The default behavior for shutdownNow shut's down the underlying s3 client - // which will cause any following s3 operations to fail. - tx.shutdownNow(false); - - // copy to [name]-latest.zip - String copyKey = getLatestS3BundleKey(); - CopyObjectRequest copyObjRequest = new CopyObjectRequest( - otpServer.s3Bucket, key, otpServer.s3Bucket, copyKey); - FeedStore.s3Client.copyObject(copyObjRequest); - LOG.info("Copied to s3://{}/{}", otpServer.s3Bucket, copyKey); - } catch (AmazonClientException|InterruptedException e) { - statusMessage = String.format("Error uploading (or copying) deployment bundle to s3://%s/%s", otpServer.s3Bucket, key); - LOG.error(statusMessage); - e.printStackTrace(); + uploadBundleToS3(); + } catch (AmazonClientException | InterruptedException e) { + statusMessage = String.format("Error uploading (or copying) deployment bundle to s3://%s", s3Bucket); + LOG.error(statusMessage, e); status.fail(statusMessage); - return; } - LOG.info("Uploaded to s3://{}/{}", otpServer.s3Bucket, getS3BundleKey()); - status.update("Upload to S3 complete.", status.percentComplete + 10); - status.uploadingS3 = false; + } + // Handle spinning up new EC2 servers for the load balancer's target group. if (otpServer.targetGroupArn != null) { if ("true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled"))) { - createServer(); + replaceEC2Servers(); // If creating a new server, there is no need to deploy to an existing one. return; } else { @@ -209,11 +189,46 @@ public void jobLogic () { } // If there are no OTP targets (i.e. we're only deploying to S3), we're done. - if(otpServer.internalUrl == null) { - status.completed = true; - return; + if(otpServer.internalUrl != null) { + // If we come to this point, there are internal URLs we need to deploy to (i.e., build graph over the wire). + boolean sendOverWireSuccessful = buildGraphOverWire(); + if (!sendOverWireSuccessful) return; + // Set baseUrl after success. + status.baseUrl = otpServer.publicUrl; } + status.completed = true; + } + private void uploadBundleToS3() throws InterruptedException, AmazonClientException { + status.message = "Uploading to S3"; + status.uploadingS3 = true; + String key = getS3BundleKey(); + LOG.info("Uploading deployment {} to s3://{}/{}", deployment.name, s3Bucket, key); + TransferManager tx = TransferManagerBuilder.standard().withS3Client(FeedStore.s3Client).build(); + final Upload upload = tx.upload(s3Bucket, key, deploymentTempFile); + + upload.addProgressListener( + (ProgressListener) progressEvent -> status.percentUploaded = upload.getProgress().getPercentTransferred() + ); + + upload.waitForCompletion(); + + // Shutdown the Transfer Manager, but don't shut down the underlying S3 client. + // The default behavior for shutdownNow shut's down the underlying s3 client + // which will cause any following s3 operations to fail. + tx.shutdownNow(false); + + // copy to [name]-latest.zip + String copyKey = getLatestS3BundleKey(); + CopyObjectRequest copyObjRequest = new CopyObjectRequest(s3Bucket, key, s3Bucket, copyKey); + FeedStore.s3Client.copyObject(copyObjRequest); + LOG.info("Copied to s3://{}/{}", s3Bucket, copyKey); + LOG.info("Uploaded to s3://{}/{}", s3Bucket, getS3BundleKey()); + status.update("Upload to S3 complete.", status.percentComplete + 10); + status.uploadingS3 = false; + } + + private boolean buildGraphOverWire() { // figure out what router we're using String router = deployment.routerId != null ? deployment.routerId : "default"; @@ -267,7 +282,7 @@ public void jobLogic () { LOG.error(statusMessage); e.printStackTrace(); status.fail(statusMessage); - return; + return false; } // retrieveById the input file @@ -277,7 +292,7 @@ public void jobLogic () { } catch (FileNotFoundException e) { LOG.error("Internal error: could not read dumped deployment!"); status.fail("Internal error: could not read dumped deployment!"); - return; + return false; } try { @@ -286,7 +301,7 @@ public void jobLogic () { statusMessage = String.format("Unable to open connection to OTP server %s", url); LOG.error(statusMessage); status.fail(statusMessage); - return; + return false; } // copy @@ -297,7 +312,7 @@ public void jobLogic () { LOG.error(statusMessage); e.printStackTrace(); status.fail(statusMessage); - return; + return false; } try { @@ -307,7 +322,7 @@ public void jobLogic () { LOG.error(message); e.printStackTrace(); status.fail(message); - return; + return false; } try { @@ -326,8 +341,8 @@ public void jobLogic () { if (code != HttpURLConnection.HTTP_CREATED) { // Get input/error stream from connection response. InputStream stream = code < HttpURLConnection.HTTP_BAD_REQUEST - ? conn.getInputStream() - : conn.getErrorStream(); + ? conn.getInputStream() + : conn.getErrorStream(); String response; try (Scanner scanner = new Scanner(stream)) { scanner.useDelimiter("\\Z"); @@ -338,7 +353,7 @@ public void jobLogic () { status.fail(statusMessage); // Skip deploying to any other servers. // There is no reason to take out the rest of the servers, it's going to have the same result. - return; + return false; } } catch (IOException e) { statusMessage = String.format("Could not finish request to server %s", url); @@ -350,9 +365,7 @@ public void jobLogic () { tasksCompleted++; status.percentComplete = 100.0 * (double) tasksCompleted / totalTasks; } - - status.completed = true; - status.baseUrl = otpServer.publicUrl; + return true; } private String getS3BundleKey() { @@ -388,7 +401,7 @@ public void jobFinished () { NotifyUsersForSubscriptionJob.createNotification("deployment-updated", deployment.id, message); } - public void createServer () { + public void replaceEC2Servers() { try { // First start graph-building instance and wait for graph to successfully build. List instances = startEC2Instances(1); @@ -460,7 +473,7 @@ private List startEC2Instances(int count) { // 2. Time to live until shutdown/termination (for test servers) // 3. Hosting / nginx // FIXME: Allow for r5 servers to be created. - String userData = constructUserData(otpServer.s3Bucket, deployment.r5); + String userData = constructUserData(deployment.r5); // The subnet ID should only change if starting up a server in some other AWS account. This is not // likely to be a requirement. // Define network interface so that a public IP can be associated with server. @@ -538,7 +551,7 @@ private List getIds (List instances) { return instances.stream().map(Instance::getInstanceId).collect(Collectors.toList()); } - private String constructUserData(String s3Bucket, boolean r5) { + private String constructUserData(boolean r5) { // Prefix/name of JAR file (WITHOUT .jar) FIXME: make this configurable. String jarName = r5 ? deployment.r5Version : deployment.otpVersion; if (jarName == null) { @@ -563,12 +576,6 @@ private String constructUserData(String s3Bucket, boolean r5) { lines.add(String.format("LOGFILE=/var/log/%s.log", tripPlanner)); // Log user data setup to /var/log/user-data.log lines.add("exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1"); - // Install the necessary files to run the trip planner. FIXME Remove. This should be captured by the AMI. -// lines.add("apt update -y"); -// lines.add("apt install -y openjdk-8-jre"); -// lines.add("apt install -y openjfx"); -// lines.add("apt install -y awscli"); -// lines.add("apt install -y unzip"); // Create the directory for the graph inputs. lines.add(String.format("mkdir -p %s", routerDir)); lines.add(String.format("chown ubuntu %s", routerDir)); From 6ae205569ab3625bb65570164b8105927a5fe254 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 7 Aug 2019 17:49:53 -0400 Subject: [PATCH 13/42] refactor: fix issues resulting from merge --- .../controllers/api/DeploymentController.java | 9 ++------- .../controllers/api/ProjectController.java | 5 +---- .../controllers/api/ServerController.java | 18 +++++++++--------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 385db49c1..f5b45d65c 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -252,13 +252,8 @@ private static String deploy (Request req, Response res) { Project project = Persistence.projects.getById(deployment.projectId); if (project == null) logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); - - // FIXME: Currently the otp server to deploy to is determined by the string name field (with special characters - // replaced with underscores). This should perhaps be replaced with an immutable server ID so that there is - // no risk that these values can overlap. This may be over engineering this system though. The user deploying - // a set of feeds would likely not create two deployment targets with the same name (and the name is unlikely - // to change often). - OtpServer otpServer = project.retrieveServer(target); + // Get server by ID + OtpServer otpServer = Persistence.servers.getById(target); if (otpServer == null) logMessageAndHalt(req, 400, "Must provide valid OTP server target ID."); // Check that permissions of user allow them to deploy to target. diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java index cc4db1c1c..5bc91691b 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java @@ -120,10 +120,7 @@ private static Project updateProject(Request req, Response res) { private static Project deleteProject(Request req, Response res) { // Fetch project first to check permissions, and so we can return the deleted project after deletion. Project project = requestProjectById(req, "manage"); - boolean successfullyDeleted = project.delete(); - if (!successfullyDeleted) { - logMessageAndHalt(req, 500, "Did not delete project.", new Exception("Delete unsuccessful")); - } + project.delete(); return project; } diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 0ae5c8704..9e05b47f9 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -22,7 +22,7 @@ import java.util.Collections; import java.util.List; -import static com.conveyal.datatools.common.utils.SparkUtils.haltWithMessage; +import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static spark.Spark.delete; import static spark.Spark.get; import static spark.Spark.options; @@ -46,12 +46,12 @@ private static OtpServer checkServerPermissions(Request req, Response res) { String serverId = req.params("id"); OtpServer server = Persistence.servers.getById(serverId); if (server == null) { - haltWithMessage(req, HttpStatus.BAD_REQUEST_400, "Server does not exist."); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server does not exist."); } boolean isProjectAdmin = userProfile.canAdministerProject(server.projectId, server.organizationId()); if (!isProjectAdmin && !userProfile.getUser_id().equals(server.user())) { // If user is not a project admin and did not create the deployment, access to the deployment is denied. - haltWithMessage(req, HttpStatus.UNAUTHORIZED_401, "User not authorized for deployment."); + logMessageAndHalt(req, HttpStatus.UNAUTHORIZED_401, "User not authorized for deployment."); } return server; } @@ -88,7 +88,7 @@ private static OtpServer createServer(Request req, Response res) { Persistence.servers.create(newServer); return Persistence.servers.update(newServer.id, req.body()); } else { - haltWithMessage(req, 403, "Not authorized to create a server for project " + projectId); + logMessageAndHalt(req, 403, "Not authorized to create a server for project " + projectId); return null; } } @@ -102,7 +102,7 @@ private static List fetchServers (Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); if (projectId != null) { Project project = Persistence.projects.getById(projectId); - if (project == null) haltWithMessage(req, 400, "Must provide a valid project ID."); + if (project == null) logMessageAndHalt(req, 400, "Must provide a valid project ID."); else if (userProfile.canAdministerProject(projectId, null)) return project.availableOtpServers(); } else if (userProfile.canAdministerApplication()) return Persistence.servers.getAll(); @@ -118,7 +118,7 @@ private static OtpServer updateServer(Request req, Response res) { Document updateDocument = Document.parse(req.body()); Auth0UserProfile user = req.attribute("user"); if ((serverToUpdate.admin || serverToUpdate.projectId == null) && !user.canAdministerApplication()) { - haltWithMessage(req, 401, "User cannot modify admin-only or application-wide server."); + logMessageAndHalt(req, 401, "User cannot modify admin-only or application-wide server."); } validateFields(req, updateDocument); OtpServer updatedServer = Persistence.servers.update(serverToUpdate.id, updateDocument); @@ -138,7 +138,7 @@ private static void validateFields(Request req, Document serverDocument) throws serverDocument.remove("dateCreated"); if (serverDocument.containsKey("projectId") && serverDocument.get("projectId") != null) { Project project = Persistence.projects.getById(serverDocument.get("projectId").toString()); - if (project == null) haltWithMessage(req, 400, "Must specify valid project ID."); + if (project == null) logMessageAndHalt(req, 400, "Must specify valid project ID."); } if (serverDocument.containsKey("targetGroupArn") && serverDocument.get("targetGroupArn") != null) { // Validate that the Target Group ARN is valid. @@ -148,10 +148,10 @@ private static void validateFields(Request req, Document serverDocument) throws AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); List targetGroups = elb.describeTargetGroups(describeTargetGroupsRequest).getTargetGroups(); if (targetGroups.size() == 0) { - haltWithMessage(req, 400, "Invalid value for Target Group ARN. Could not locate Target Group."); + logMessageAndHalt(req, 400, "Invalid value for Target Group ARN. Could not locate Target Group."); } } catch (AmazonElasticLoadBalancingException e) { - haltWithMessage(req, 400, "Invalid value for Target Group ARN."); + logMessageAndHalt(req, 400, "Invalid value for Target Group ARN."); } } } From c33c290565613e6bca13f463933589e2edb9ead6 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 8 Aug 2019 17:13:08 -0400 Subject: [PATCH 14/42] refactor(deployment): tweak user script and update default config --- configurations/default/server.yml.tmp | 9 +++++++++ .../datatools/manager/jobs/DeployJob.java | 19 ++++++------------- .../manager/jobs/MonitorServerStatusJob.java | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index c29382e26..fee44236d 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -13,6 +13,15 @@ modules: enabled: false editor: enabled: false + deployment: + enabled: false + ec2: + enabled: false + subnet: subnet-id + securityGroup: security-group-id + ami: ami-id + arn: arn + keyName: some-pem-file-without-suffix user_admin: enabled: true # Enable GTFS+ module for testing purposes diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 8621e0b80..a5643fb43 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -170,10 +170,6 @@ public void jobLogic () { status.fail(message); return; } - status.message = "Uploading to S3"; - status.uploadingS3 = true; - LOG.info("Uploading deployment {} to s3", deployment.name); - String key = null; try { uploadBundleToS3(); } catch (AmazonClientException | InterruptedException e) { @@ -210,7 +206,7 @@ public void jobLogic () { } private void uploadBundleToS3() throws InterruptedException, AmazonClientException { - status.message = "Uploading to S3"; + status.message = "Uploading to s3://" + s3Bucket; status.uploadingS3 = true; String key = getS3BundleKey(); LOG.info("Uploading deployment {} to s3://{}/{}", deployment.name, s3Bucket, key); @@ -562,7 +558,7 @@ private List getIds (List instances) { } private String constructUserData(boolean r5) { - // Prefix/name of JAR file (WITHOUT .jar) FIXME: make this configurable. + // Prefix/name of JAR file (WITHOUT .jar) String jarName = r5 ? deployment.r5Version : deployment.otpVersion; if (jarName == null) { // If there is no version specified, use the default (and persist value). @@ -571,8 +567,8 @@ private String constructUserData(boolean r5) { } String tripPlanner = r5 ? "r5" : "otp"; String s3JarBucket = r5 ? "r5-builds" : "opentripplanner-builds"; - String s3JarUrl = String.format("s3://%s/%s.jar", s3JarBucket, jarName); - // FIXME Check that jar URL exists. + String s3JarUrl = String.format("https://%s.s3.amazonaws.com/%s.jar", s3JarBucket, jarName); + // TODO Check that jar URL exists? String jarDir = String.format("/opt/%s", tripPlanner); String s3BundlePath = String.format("s3://%s/%s", s3Bucket, getS3BundleKey()); boolean graphAlreadyBuilt = FeedStore.s3Client.doesObjectExist(s3Bucket, getS3GraphKey()); @@ -600,10 +596,7 @@ private String constructUserData(boolean r5) { } // Download trip planner JAR. lines.add(String.format("mkdir -p %s", jarDir)); - String region = deployment.r5 ? "eu-west-1" : "us-east-1"; - lines.add(String.format("aws s3 cp --region %s %s %s/%s.jar", region, s3JarUrl, jarDir, jarName)); - // Kill any running java process FIXME Currently the AMI we're using starts up OTP on startup, so we need to kill the java process in order to follow the below instructions - lines.add("sudo pkill -9 -f java"); + lines.add(String.format("wget %s -O %s/%s.jar", s3JarUrl, jarDir, jarName)); if (graphAlreadyBuilt) { lines.add("echo 'downloading graph from s3'"); // Download Graph from S3 and spin up trip planner. @@ -621,7 +614,7 @@ private String constructUserData(boolean r5) { lines.add("sudo poweroff"); } else { lines.add("echo 'kicking off trip planner (logs at $LOGFILE)'"); - // Kick off the application. FIXME use sudo service opentripplanner start? + // Kick off the application. if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu nohup java -Xmx6G -Djava.util.Arrays.useLegacyMergeSort=true -jar %s/%s.jar point --isochrones %s > /var/log/r5.out 2>&1&", jarDir, jarName, routerDir)); else lines.add(String.format("sudo -H -u ubuntu nohup java -jar %s/%s.jar --server --bindAddress 127.0.0.1 --router default > $LOGFILE 2>&1 &", jarDir, jarName)); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 0928c4c49..23d366a87 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -59,7 +59,7 @@ public void jobLogic() { // FeedStore.s3Client.doesObjectExist(otpServer.s3Bucket, ); } // First, check that OTP has started up. - status.update("Instance status is OK.", 20); + status.update("Instance status is OK. Waiting for OTP...", 20); while (!otpIsRunning) { LOG.info("Checking that OTP is running on server at {}.", otpUrl); // If the request is successful, the OTP instance has started. From 05ec4df1dfa3b3be48c7070592ce0f75af462149 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 9 Aug 2019 11:40:04 -0400 Subject: [PATCH 15/42] refactor(deployment): improve validation of server fields --- .../controllers/api/ServerController.java | 67 +++++++++++++++---- .../datatools/manager/jobs/DeployJob.java | 2 + 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 9e05b47f9..215701b53 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -1,16 +1,20 @@ package com.conveyal.datatools.manager.controllers.api; +import com.amazonaws.services.ec2.model.InstanceType; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException; import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetGroupsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.TargetGroup; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.json.JsonManager; +import com.fasterxml.jackson.databind.ObjectMapper; import org.bson.Document; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; @@ -19,10 +23,15 @@ import spark.Request; import spark.Response; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.util.Collections; import java.util.List; +import java.util.UUID; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; +import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE; import static spark.Spark.delete; import static spark.Spark.get; import static spark.Spark.options; @@ -36,6 +45,7 @@ public class ServerController { private static JsonManager json = new JsonManager<>(OtpServer.class, JsonViews.UserInterface.class); private static final Logger LOG = LoggerFactory.getLogger(ServerController.class); + private static final ObjectMapper mapper = new ObjectMapper(); /** * Gets the server specified by the request's id parameter and ensure that user has access to the @@ -67,8 +77,6 @@ private static OtpServer deleteServer(Request req, Response res) { * deployment. */ private static OtpServer createServer(Request req, Response res) { - // TODO error handling when request is bogus - // TODO factor out user profile fetching, permissions checks etc. Auth0UserProfile userProfile = req.attribute("user"); Document newServerFields = Document.parse(req.body()); String projectId = newServerFields.getString("projectId"); @@ -78,15 +86,17 @@ private static OtpServer createServer(Request req, Response res) { boolean allowedToCreate = projectId == null ? userProfile.canAdministerApplication() : userProfile.canAdministerProject(projectId, organizationId); - if (allowedToCreate) { - OtpServer newServer = new OtpServer(); validateFields(req, newServerFields); - // FIXME: Here we are creating a deployment and updating it with the JSON string (two db operations) - // We do this because there is not currently apply JSON directly to an object (outside of Mongo codec - // operations) + OtpServer newServer; + try { + newServer = mapper.readValue(newServerFields.toJson(), OtpServer.class); + } catch (IOException e) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Error parsing OTP server JSON."); + return null; + } Persistence.servers.create(newServer); - return Persistence.servers.update(newServer.id, req.body()); + return newServer; } else { logMessageAndHalt(req, 403, "Not authorized to create a server for project " + projectId); return null; @@ -118,7 +128,7 @@ private static OtpServer updateServer(Request req, Response res) { Document updateDocument = Document.parse(req.body()); Auth0UserProfile user = req.attribute("user"); if ((serverToUpdate.admin || serverToUpdate.projectId == null) && !user.canAdministerApplication()) { - logMessageAndHalt(req, 401, "User cannot modify admin-only or application-wide server."); + logMessageAndHalt(req, HttpStatus.UNAUTHORIZED_401, "User cannot modify admin-only or application-wide server."); } validateFields(req, updateDocument); OtpServer updatedServer = Persistence.servers.update(serverToUpdate.id, updateDocument); @@ -136,10 +146,12 @@ private static void validateFields(Request req, Document serverDocument) throws // speaks to the fragility of this system currently. serverDocument.remove("lastUpdated"); serverDocument.remove("dateCreated"); + // Check that projectId is valid. if (serverDocument.containsKey("projectId") && serverDocument.get("projectId") != null) { Project project = Persistence.projects.getById(serverDocument.get("projectId").toString()); - if (project == null) logMessageAndHalt(req, 400, "Must specify valid project ID."); + if (project == null) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Must specify valid project ID."); } + // If server has a target group specified, it must have a few other fields as well (e.g., instance type). if (serverDocument.containsKey("targetGroupArn") && serverDocument.get("targetGroupArn") != null) { // Validate that the Target Group ARN is valid. try { @@ -148,10 +160,41 @@ private static void validateFields(Request req, Document serverDocument) throws AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); List targetGroups = elb.describeTargetGroups(describeTargetGroupsRequest).getTargetGroups(); if (targetGroups.size() == 0) { - logMessageAndHalt(req, 400, "Invalid value for Target Group ARN. Could not locate Target Group."); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group."); } } catch (AmazonElasticLoadBalancingException e) { - logMessageAndHalt(req, 400, "Invalid value for Target Group ARN."); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); + } + // Validate instance type + if (serverDocument.get("instanceType") != null) { + try { + InstanceType.fromValue(serverDocument.get("instanceType").toString()); + } catch (IllegalArgumentException e) { + String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + } + } + } + // Server must have name. + if (serverDocument.get("name") == null || "".equals(serverDocument.get("name").toString())) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must have valid name."); + } + // Server must have an internal URL (for build graph over wire) or an s3 bucket (for auto deploy ec2). + if (serverDocument.get("s3Bucket") == null || "".equals(serverDocument.get("s3Bucket").toString())) { + if (!serverDocument.containsKey("internalUrl") || ((List) serverDocument.get("internalUrl")).size() == 0) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must contain either internal URL(s) or s3 bucket name."); + } + } else { + // Verify that application has permission to write to/delete from S3 bucket. + String key = UUID.randomUUID().toString(); + String bucket = serverDocument.get("s3Bucket").toString(); + try { + FeedStore.s3Client.putObject(bucket, key, File.createTempFile("test", ".zip")); + FeedStore.s3Client.deleteObject(bucket, key); + } catch (IOException | AmazonS3Exception e) { + String message = "Cannot write to specified S3 bucket " + bucket; + LOG.error(message, e); + logMessageAndHalt(req, 400, message); } } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index a5643fb43..df3f2f1c5 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -73,6 +73,7 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); private static final String bundlePrefix = "bundles"; + public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; /** * S3 bucket to upload deployment to. If not null, uses {@link OtpServer#s3Bucket}. Otherwise, defaults to * {@link DataManager#feedBucket} @@ -474,6 +475,7 @@ public void replaceEC2Servers() { } private List startEC2Instances(int count) { + String instanceType = otpServer.instanceType == null ? DEFAULT_INSTANCE_TYPE : otpServer.instanceType; // User data should contain info about: // 1. Downloading GTFS/OSM info (s3) // 2. Time to live until shutdown/termination (for test servers) From b6d73632daea759cc0bc63c3da9b44dc75148a00 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 9 Aug 2019 13:48:42 -0400 Subject: [PATCH 16/42] refactor(deployments): modify OtpServer fields and refactor server creation --- .../manager/controllers/api/ServerController.java | 7 +++---- .../com/conveyal/datatools/manager/models/OtpServer.java | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 215701b53..c761e2dc3 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -88,15 +88,14 @@ private static OtpServer createServer(Request req, Response res) { : userProfile.canAdministerProject(projectId, organizationId); if (allowedToCreate) { validateFields(req, newServerFields); - OtpServer newServer; try { - newServer = mapper.readValue(newServerFields.toJson(), OtpServer.class); + OtpServer newServer = mapper.readValue(newServerFields.toJson(), OtpServer.class); + Persistence.servers.create(newServer); + return newServer; } catch (IOException e) { logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Error parsing OTP server JSON."); return null; } - Persistence.servers.create(newServer); - return newServer; } else { logMessageAndHalt(req, 403, "Not authorized to create a server for project " + projectId); return null; diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java index eef5c6693..d24dc8946 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java @@ -20,8 +20,6 @@ public class OtpServer extends Model { public String publicUrl; public boolean admin; public String s3Bucket; - public String s3Credentials; - public boolean createServer; /** Empty constructor for serialization. */ public OtpServer () {} From 7a020c834b9ab6b6acc013a67d8908e76ba760a0 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 9 Aug 2019 13:50:07 -0400 Subject: [PATCH 17/42] refactor: remove unused import --- .../datatools/manager/controllers/api/ServerController.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index c761e2dc3..334fea4ed 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -25,7 +25,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Collections; import java.util.List; import java.util.UUID; From c753a31349b6e3816a47fdb41c63bf7b6d71da7a Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 13 Aug 2019 08:36:24 -0400 Subject: [PATCH 18/42] refactor(ServerController): add comment about checking S3 permissions --- .../datatools/manager/controllers/api/ServerController.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 334fea4ed..ed5846c89 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -183,7 +183,10 @@ private static void validateFields(Request req, Document serverDocument) throws logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must contain either internal URL(s) or s3 bucket name."); } } else { - // Verify that application has permission to write to/delete from S3 bucket. + // Verify that application has permission to write to/delete from S3 bucket. We're following the recommended + // approach from https://stackoverflow.com/a/17284647/915811, but perhaps there is a way to do this + // effectively without incurring AWS costs (although writing/deleting an empty file to S3 is probably + // miniscule). String key = UUID.randomUUID().toString(); String bucket = serverDocument.get("s3Bucket").toString(); try { From b7937273e37590baf8a267426d7218940603cbd4 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Wed, 14 Aug 2019 17:31:28 -0400 Subject: [PATCH 19/42] refactor(ServerController): add missing exceptions to logMessageAndHalt --- .../manager/controllers/api/ServerController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index ed5846c89..6bfb2c928 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -92,7 +92,7 @@ private static OtpServer createServer(Request req, Response res) { Persistence.servers.create(newServer); return newServer; } catch (IOException e) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Error parsing OTP server JSON."); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Error parsing OTP server JSON.", e); return null; } } else { @@ -161,7 +161,7 @@ private static void validateFields(Request req, Document serverDocument) throws logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group."); } } catch (AmazonElasticLoadBalancingException e) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN.", e); } // Validate instance type if (serverDocument.get("instanceType") != null) { @@ -169,7 +169,7 @@ private static void validateFields(Request req, Document serverDocument) throws InstanceType.fromValue(serverDocument.get("instanceType").toString()); } catch (IllegalArgumentException e) { String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE); - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e); } } } @@ -195,7 +195,7 @@ private static void validateFields(Request req, Document serverDocument) throws } catch (IOException | AmazonS3Exception e) { String message = "Cannot write to specified S3 bucket " + bucket; LOG.error(message, e); - logMessageAndHalt(req, 400, message); + logMessageAndHalt(req, 400, message, e); } } } From a3ed73c34db9f32f9232d365243d5f2f9cd063f0 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 20 Aug 2019 11:44:46 -0400 Subject: [PATCH 20/42] refactor(deploy): fix check for s3 graph object --- .../conveyal/datatools/manager/jobs/DeployJob.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index df3f2f1c5..9dc68526a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -19,6 +19,7 @@ import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsResult; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @@ -573,7 +574,15 @@ private String constructUserData(boolean r5) { // TODO Check that jar URL exists? String jarDir = String.format("/opt/%s", tripPlanner); String s3BundlePath = String.format("s3://%s/%s", s3Bucket, getS3BundleKey()); - boolean graphAlreadyBuilt = FeedStore.s3Client.doesObjectExist(s3Bucket, getS3GraphKey()); + // Note, an AmazonS3Exception will be thrown by S3 if the object does not exist. This is a feature to avoid + // revealing to non-authorized users whether the object actually exists. So, as long as permissions are + // configured correctly a 403 indicates it does not exist. + boolean graphAlreadyBuilt = false; + try{ + graphAlreadyBuilt = FeedStore.s3Client.doesObjectExist(s3Bucket, getS3GraphKey()); + } catch (AmazonS3Exception e) { + LOG.warn("Error checking if graph object exists. This is likely because it does not exist.", e); + } List lines = new ArrayList<>(); String routerName = "default"; String routerDir = String.format("/var/%s/graphs/%s", tripPlanner, routerName); From a07935a8ba14fe621b5965a52c876b85c834ef4e Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 20 Aug 2019 14:51:59 -0400 Subject: [PATCH 21/42] refactor(deploy): revert to default instance type if none specified --- .../java/com/conveyal/datatools/manager/jobs/DeployJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 9dc68526a..3dddfab71 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -494,7 +494,7 @@ private List startEC2Instances(int count) { RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) - .withInstanceType(otpServer.instanceType) + .withInstanceType(instanceType) .withMinCount(count) .withMaxCount(count) .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(DataManager.getConfigPropertyAsText("modules.deployment.ec2.arn"))) From a177e7990d864989f5209d90e1da3b222e5365d1 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 20 Aug 2019 15:14:48 -0400 Subject: [PATCH 22/42] refactor(deploy): make instance profile arn optional --- .../com/conveyal/datatools/manager/jobs/DeployJob.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 3dddfab71..2c6e9e16d 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -497,12 +497,17 @@ private List startEC2Instances(int count) { .withInstanceType(instanceType) .withMinCount(count) .withMaxCount(count) - .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(DataManager.getConfigPropertyAsText("modules.deployment.ec2.arn"))) .withImageId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ami")) .withKeyName(DataManager.getConfigPropertyAsText("modules.deployment.ec2.keyName")) // This will have the instance terminate when it is shut down. .withInstanceInitiatedShutdownBehavior("terminate") .withUserData(Base64.encodeBase64String(userData.getBytes())); + // Add instance profile if specified. + if (DataManager.hasConfigProperty("modules.deployment.ec2.arn")) { + IamInstanceProfileSpecification instanceProfile = new IamInstanceProfileSpecification() + .withArn(DataManager.getConfigPropertyAsText("modules.deployment.ec2.arn")); + runInstancesRequest.withIamInstanceProfile(instanceProfile); + } final List instances = ec2.runInstances(runInstancesRequest).getReservation().getInstances(); List instanceIds = getIds(instances); From 2507ab5f3a3f8c58ea2d52c70c01d987dcf7f4f3 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 22 Aug 2019 09:56:22 -0400 Subject: [PATCH 23/42] refactor(deploy): use set method rather than with for instance profile --- .../java/com/conveyal/datatools/manager/jobs/DeployJob.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 2c6e9e16d..e590293e1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -502,11 +502,11 @@ private List startEC2Instances(int count) { // This will have the instance terminate when it is shut down. .withInstanceInitiatedShutdownBehavior("terminate") .withUserData(Base64.encodeBase64String(userData.getBytes())); - // Add instance profile if specified. + // Set IAM instance profile if specified. if (DataManager.hasConfigProperty("modules.deployment.ec2.arn")) { IamInstanceProfileSpecification instanceProfile = new IamInstanceProfileSpecification() .withArn(DataManager.getConfigPropertyAsText("modules.deployment.ec2.arn")); - runInstancesRequest.withIamInstanceProfile(instanceProfile); + runInstancesRequest.setIamInstanceProfile(instanceProfile); } final List instances = ec2.runInstances(runInstancesRequest).getReservation().getInstances(); From 4a1ef29aa0c8955657421c8d0c64c7634ad598a0 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 9 Sep 2019 19:16:27 -0400 Subject: [PATCH 24/42] refactor(deploy): move ec2 config into OtpServer --- configurations/default/server.yml.tmp | 5 - pom.xml | 10 +- .../datatools/common/utils/SparkUtils.java | 9 +- .../controllers/api/DeploymentController.java | 38 +++ .../controllers/api/ServerController.java | 221 ++++++++++++++---- .../datatools/manager/jobs/DeployJob.java | 153 +++++++----- .../manager/jobs/MonitorServerStatusJob.java | 20 +- .../datatools/manager/models/Deployment.java | 17 +- .../datatools/manager/models/EC2Info.java | 36 +++ .../manager/models/EC2InstanceSummary.java | 56 +++++ .../datatools/manager/models/OtpServer.java | 44 +++- 11 files changed, 470 insertions(+), 139 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/EC2Info.java create mode 100644 src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index 61b17160a..16249fe4d 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -19,11 +19,6 @@ modules: enabled: false ec2: enabled: false - subnet: subnet-id - securityGroup: security-group-id - ami: ami-id - arn: arn - keyName: some-pem-file-without-suffix user_admin: enabled: true # Enable GTFS+ module for testing purposes diff --git a/pom.xml b/pom.xml index a7525e553..4f7b0e06d 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ 17.5 + 1.11.625 @@ -398,12 +399,17 @@ com.amazonaws aws-java-sdk-ec2 - 1.11.410 + ${awsjavasdk.version} + + + com.amazonaws + aws-java-sdk-iam + ${awsjavasdk.version} com.amazonaws aws-java-sdk-elasticloadbalancingv2 - 1.11.410 + ${awsjavasdk.version} diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index f69c11f5e..8f4428cfb 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -122,7 +122,7 @@ public static void logMessageAndHalt( if (bugsnag != null && e != null) { // create report to send to bugsnag Report report = bugsnag.buildReport(e); - Auth0UserProfile userProfile = request.attribute("user"); + Auth0UserProfile userProfile = request != null ? request.attribute("user") : null; String userEmail = userProfile != null ? userProfile.getEmail() : "no-auth"; report.setUserEmail(userEmail); bugsnag.notify(report); @@ -218,11 +218,16 @@ public static void logRequestOrResponse( String bodyString, int statusCode ) { + // If request is null, log warning and exit. We do not want to hit an NPE in this method. + if (request == null) { + LOG.warn("Request object is null. Cannot log."); + return; + } Auth0UserProfile userProfile = request.attribute("user"); String userEmail = userProfile != null ? userProfile.getEmail() : "no-auth"; String queryString = request.queryParams().size() > 0 ? "?" + request.queryString() : ""; LOG.info( - "{} {} {}: {}{}{}{}", + "{} {} {}: {}{}{} {}", logRequest ? "req" : String.format("res (%s)", statusCode), userEmail, request.requestMethod(), diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index f5b45d65c..d3bc5ed11 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -1,10 +1,18 @@ package com.conveyal.datatools.manager.controllers.api; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; +import com.amazonaws.services.ec2.model.DescribeInstancesResult; +import com.amazonaws.services.ec2.model.Filter; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Reservation; import com.conveyal.datatools.common.utils.SparkUtils; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.DeployJob; import com.conveyal.datatools.manager.models.Deployment; +import com.conveyal.datatools.manager.models.EC2InstanceSummary; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.JsonViews; @@ -24,6 +32,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +53,7 @@ public class DeploymentController { private static JsonManager json = new JsonManager<>(Deployment.class, JsonViews.UserInterface.class); private static final Logger LOG = LoggerFactory.getLogger(DeploymentController.class); private static Map deploymentJobsByServer = new HashMap<>(); + private static final AmazonEC2 ec2 = AmazonEC2Client.builder().build(); /** * Gets the deployment specified by the request's id parameter and ensure that user has access to the @@ -241,6 +251,33 @@ private static Object updateDeployment (Request req, Response res) { return updatedDeployment; } + /** + * HTTP controller to fetch information about provided EC2 machines that power ELBs running a trip planner. + */ + private static List fetchEC2InstanceSummaries(Request req, Response res) { + Deployment deployment = checkDeploymentPermissions(req, res); + return deployment.retrieveEC2Instances(); + } + + /** + * Fetches list of {@link EC2InstanceSummary} for all instances matching the provided filters. + */ + public static List fetchEC2InstanceSummaries(Filter... filters) { + return fetchEC2Instances(filters).stream().map(EC2InstanceSummary::new).collect(Collectors.toList()); + } + + public static List fetchEC2Instances(Filter... filters) { + List instances = new ArrayList<>(); + DescribeInstancesRequest request = new DescribeInstancesRequest().withFilters(filters); + DescribeInstancesResult result = ec2.describeInstances(request); + for (Reservation reservation : result.getReservations()) { + instances.addAll(reservation.getInstances()); + } + // Sort by launch time (most recent first). + instances.sort(Comparator.comparing(Instance::getLaunchTime).reversed()); + return instances; + } + /** * Create a deployment bundle, and send it to the specified OTP target servers (or the specified s3 bucket). */ @@ -313,6 +350,7 @@ public static void register (String apiPrefix) { }), json::write); options(apiPrefix + "secure/deployments", (q, s) -> ""); get(apiPrefix + "secure/deployments/:id/download", DeploymentController::downloadDeployment); + get(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::fetchEC2InstanceSummaries, json::write); get(apiPrefix + "secure/deployments/:id", DeploymentController::getDeployment, json::write); delete(apiPrefix + "secure/deployments/:id", DeploymentController::deleteDeployment, json::write); get(apiPrefix + "secure/deployments", DeploymentController::getAllDeployments, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 6bfb2c928..11b109fe2 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -1,13 +1,35 @@ package com.conveyal.datatools.manager.controllers.api; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.AmazonEC2Exception; +import com.amazonaws.services.ec2.model.DescribeImagesRequest; +import com.amazonaws.services.ec2.model.DescribeImagesResult; +import com.amazonaws.services.ec2.model.DescribeKeyPairsResult; +import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest; +import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult; +import com.amazonaws.services.ec2.model.DescribeSubnetsRequest; +import com.amazonaws.services.ec2.model.DescribeSubnetsResult; +import com.amazonaws.services.ec2.model.Image; +import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.InstanceType; +import com.amazonaws.services.ec2.model.KeyPairInfo; +import com.amazonaws.services.ec2.model.SecurityGroup; +import com.amazonaws.services.ec2.model.Subnet; +import com.amazonaws.services.ec2.model.TerminateInstancesRequest; +import com.amazonaws.services.ec2.model.TerminateInstancesResult; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException; import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetGroupsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.TargetGroup; +import com.amazonaws.services.identitymanagement.AmazonIdentityManagement; +import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClientBuilder; +import com.amazonaws.services.identitymanagement.model.InstanceProfile; +import com.amazonaws.services.identitymanagement.model.ListInstanceProfilesResult; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.models.Project; @@ -25,9 +47,11 @@ import java.io.File; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static com.conveyal.datatools.manager.jobs.DeployJob.DEFAULT_INSTANCE_TYPE; @@ -45,6 +69,9 @@ public class ServerController { private static JsonManager json = new JsonManager<>(OtpServer.class, JsonViews.UserInterface.class); private static final Logger LOG = LoggerFactory.getLogger(ServerController.class); private static final ObjectMapper mapper = new ObjectMapper(); + private static final AmazonEC2 ec2 = AmazonEC2Client.builder().build(); + private static final AmazonIdentityManagement iam = AmazonIdentityManagementClientBuilder.defaultClient(); + private static final AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); /** * Gets the server specified by the request's id parameter and ensure that user has access to the @@ -67,10 +94,39 @@ private static OtpServer checkServerPermissions(Request req, Response res) { private static OtpServer deleteServer(Request req, Response res) { OtpServer server = checkServerPermissions(req, res); + List instances = server.retrieveEC2Instances(); + if (instances.size() > 0) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Cannot delete server with active EC2 instances: " + getIds(instances)); + } server.delete(); return server; } + /** HTTP method for terminating EC2 instances associated with an ELB OTP server. */ + private static OtpServer terminateEC2Instances(Request req, Response res) { + OtpServer server = checkServerPermissions(req, res); + List instances = server.retrieveEC2Instances(); + List ids = getIds(instances); + terminateInstances(ids); + for (Deployment deployment : Deployment.retrieveDeploymentForServerAndRouterId(server.id, null)) { + Persistence.deployments.updateField(deployment.id, "deployedTo", null); + } + return server; + } + + /** + * Shorthand method for getting list of string identifiers from a list of EC2 instances. + */ + public static List getIds (List instances) { + return instances.stream().map(Instance::getInstanceId).collect(Collectors.toList()); + } + + public static TerminateInstancesResult terminateInstances(Collection instanceIds) throws AmazonEC2Exception { + LOG.info("Terminating EC2 instances {}", instanceIds); + TerminateInstancesRequest request = new TerminateInstancesRequest().withInstanceIds(instanceIds); + return ec2.terminateInstances(request); + } + /** * Create a new server for the project. All feed sources with a valid latest version are added to the new * deployment. @@ -86,9 +142,9 @@ private static OtpServer createServer(Request req, Response res) { ? userProfile.canAdministerApplication() : userProfile.canAdministerProject(projectId, organizationId); if (allowedToCreate) { - validateFields(req, newServerFields); try { OtpServer newServer = mapper.readValue(newServerFields.toJson(), OtpServer.class); + validateFields(req, newServer); Persistence.servers.create(newServer); return newServer; } catch (IOException e) { @@ -118,68 +174,52 @@ private static List fetchServers (Request req, Response res) { } /** - * Update a single server. If the server's feed versions are updated, checks to ensure that each - * version exists and is a part of the same parent project are performed before updating. + * Update a single OTP server. */ private static OtpServer updateServer(Request req, Response res) { OtpServer serverToUpdate = checkServerPermissions(req, res); - Document updateDocument = Document.parse(req.body()); + OtpServer updatedServer = null; + try { + updatedServer = mapper.readValue(req.body(), OtpServer.class); + } catch (IOException e) { + e.printStackTrace(); + } Auth0UserProfile user = req.attribute("user"); if ((serverToUpdate.admin || serverToUpdate.projectId == null) && !user.canAdministerApplication()) { logMessageAndHalt(req, HttpStatus.UNAUTHORIZED_401, "User cannot modify admin-only or application-wide server."); } - validateFields(req, updateDocument); - OtpServer updatedServer = Persistence.servers.update(serverToUpdate.id, updateDocument); - return updatedServer; + validateFields(req, updatedServer); + Persistence.servers.replace(serverToUpdate.id, updatedServer); + return Persistence.servers.getById(updatedServer.id); } /** * Validate certain fields found in the document representing a server. This also currently modifies the document by * removing problematic date fields. */ - private static void validateFields(Request req, Document serverDocument) throws HaltException { - // FIXME: There is an issue with updating a MongoDB record with JSON serialized date fields because these come - // back as integers. MongoDB writes these values into the database fine, but when it comes to converting them - // into POJOs, it throws exceptions about expecting date types. For now, we can just remove them here, but this - // speaks to the fragility of this system currently. - serverDocument.remove("lastUpdated"); - serverDocument.remove("dateCreated"); + private static void validateFields(Request req, OtpServer server) throws HaltException { // Check that projectId is valid. - if (serverDocument.containsKey("projectId") && serverDocument.get("projectId") != null) { - Project project = Persistence.projects.getById(serverDocument.get("projectId").toString()); + if (server.projectId != null) { + Project project = Persistence.projects.getById(server.projectId); if (project == null) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Must specify valid project ID."); } - // If server has a target group specified, it must have a few other fields as well (e.g., instance type). - if (serverDocument.containsKey("targetGroupArn") && serverDocument.get("targetGroupArn") != null) { - // Validate that the Target Group ARN is valid. - try { - DescribeTargetGroupsRequest describeTargetGroupsRequest = - new DescribeTargetGroupsRequest().withTargetGroupArns(serverDocument.get("targetGroupArn").toString()); - AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); - List targetGroups = elb.describeTargetGroups(describeTargetGroupsRequest).getTargetGroups(); - if (targetGroups.size() == 0) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group."); - } - } catch (AmazonElasticLoadBalancingException e) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN.", e); - } - // Validate instance type - if (serverDocument.get("instanceType") != null) { - try { - InstanceType.fromValue(serverDocument.get("instanceType").toString()); - } catch (IllegalArgumentException e) { - String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE); - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e); - } - } + // If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to + // AWS. (e.g., target group ARN and instance type). + if (server.ec2Info != null) { + validateTargetGroup(server.ec2Info.targetGroupArn, req); + validateInstanceType(server.ec2Info.instanceType, req); + validateSubnetId(server.ec2Info.subnetId, req); + validateSecurityGroupId(server.ec2Info.securityGroupId, req); + validateIamRoleArn(server.ec2Info.iamRoleArn, req); + validateKeyName(server.ec2Info.keyName, req); + validateAmiId(server.ec2Info.amiId, req); + if (server.ec2Info.instanceCount < 0) server.ec2Info.instanceCount = 0; } // Server must have name. - if (serverDocument.get("name") == null || "".equals(serverDocument.get("name").toString())) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must have valid name."); - } + if (isEmpty(server.name)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must have valid name."); // Server must have an internal URL (for build graph over wire) or an s3 bucket (for auto deploy ec2). - if (serverDocument.get("s3Bucket") == null || "".equals(serverDocument.get("s3Bucket").toString())) { - if (!serverDocument.containsKey("internalUrl") || ((List) serverDocument.get("internalUrl")).size() == 0) { + if (isEmpty(server.s3Bucket)) { + if (server.internalUrl == null || server.internalUrl.size() == 0) { logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must contain either internal URL(s) or s3 bucket name."); } } else { @@ -188,24 +228,109 @@ private static void validateFields(Request req, Document serverDocument) throws // effectively without incurring AWS costs (although writing/deleting an empty file to S3 is probably // miniscule). String key = UUID.randomUUID().toString(); - String bucket = serverDocument.get("s3Bucket").toString(); try { - FeedStore.s3Client.putObject(bucket, key, File.createTempFile("test", ".zip")); - FeedStore.s3Client.deleteObject(bucket, key); + FeedStore.s3Client.putObject(server.s3Bucket, key, File.createTempFile("test", ".zip")); + FeedStore.s3Client.deleteObject(server.s3Bucket, key); } catch (IOException | AmazonS3Exception e) { - String message = "Cannot write to specified S3 bucket " + bucket; + String message = "Cannot write to specified S3 bucket " + server.s3Bucket; LOG.error(message, e); logMessageAndHalt(req, 400, message, e); } } } + private static void validateAmiId(String amiId, Request req) { + String message = "Server must have valid AMI ID (or field must be empty)"; + if (isEmpty(amiId)) return; + if (!amiExists(amiId)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + } + + public static boolean amiExists(String amiId) { + try { + DescribeImagesRequest request = new DescribeImagesRequest().withImageIds(amiId); + DescribeImagesResult result = ec2.describeImages(request); + // Iterate over AMIs to find a matching ID. + for (Image image : result.getImages()) if (image.getImageId().equals(amiId)) return true; + } catch (AmazonEC2Exception e) { + LOG.info("AMI does not exist.", e); + } + return false; + } + + private static void validateKeyName(String keyName, Request req) { + String message = "Server must have valid key name"; + if (isEmpty(keyName)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + DescribeKeyPairsResult response = ec2.describeKeyPairs(); + for (KeyPairInfo key_pair : response.getKeyPairs()) if (key_pair.getKeyName().equals(keyName)) return; + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + } + + private static void validateIamRoleArn(String iamRoleArn, Request req) { + String message = "Server must have valid IAM role ARN"; + if (isEmpty(iamRoleArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + ListInstanceProfilesResult result = iam.listInstanceProfiles(); + // Iterate over instance profiles. If a matching ARN is found, silently return. + for (InstanceProfile profile: result.getInstanceProfiles()) if (profile.getArn().equals(iamRoleArn)) return; + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + } + + private static void validateSecurityGroupId(String securityGroupId, Request req) { + String message = "Server must have valid security group ID"; + if (isEmpty(securityGroupId)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + DescribeSecurityGroupsRequest request = new DescribeSecurityGroupsRequest().withGroupIds(securityGroupId); + DescribeSecurityGroupsResult result = ec2.describeSecurityGroups(request); + // Iterate over groups. If a matching ID is found, silently return. + for (SecurityGroup group : result.getSecurityGroups()) if (group.getGroupId().equals(securityGroupId)) return; + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + } + + private static void validateSubnetId(String subnetId, Request req) { + String message = "Server must have valid subnet ID"; + if (isEmpty(subnetId)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + try { + DescribeSubnetsRequest request = new DescribeSubnetsRequest().withSubnetIds(subnetId); + DescribeSubnetsResult result = ec2.describeSubnets(request); + // Iterate over subnets. If a matching ID is found, silently return. + for (Subnet subnet : result.getSubnets()) if (subnet.getSubnetId().equals(subnetId)) return; + } catch (AmazonEC2Exception e) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e); + } + } + + private static void validateInstanceType(String instanceType, Request req) { + if (instanceType == null) return; + try { + InstanceType.fromValue(instanceType); + } catch (IllegalArgumentException e) { + String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE); + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e); + } + } + + private static void validateTargetGroup(String targetGroupArn, Request req) { + if (isEmpty(targetGroupArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); + try { + DescribeTargetGroupsRequest request = new DescribeTargetGroupsRequest().withTargetGroupArns(targetGroupArn); + List targetGroups = elb.describeTargetGroups(request).getTargetGroups(); + if (targetGroups.size() == 0) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN. Could not locate Target Group."); + } + } catch (AmazonElasticLoadBalancingException e) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN.", e); + } + } + + public static boolean isEmpty(String val) { + return val == null || "".equals(val); + } + /** * Register HTTP methods with handler methods. */ public static void register (String apiPrefix) { options(apiPrefix + "secure/servers", (q, s) -> ""); delete(apiPrefix + "secure/servers/:id", ServerController::deleteServer, json::write); + delete(apiPrefix + "secure/servers/:id/ec2", ServerController::terminateEC2Instances, json::write); get(apiPrefix + "secure/servers", ServerController::fetchServers, json::write); post(apiPrefix + "secure/servers", ServerController::createServer, json::write); put(apiPrefix + "secure/servers/:id", ServerController::updateServer, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index e590293e1..dad7b3ff9 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -15,9 +15,9 @@ import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.Tag; import com.amazonaws.services.ec2.model.TerminateInstancesRequest; -import com.amazonaws.services.ec2.model.TerminateInstancesResult; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; +import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest; -import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsResult; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.CopyObjectRequest; @@ -53,7 +53,9 @@ import com.amazonaws.waiters.WaiterParameters; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.controllers.api.ServerController; import com.conveyal.datatools.manager.models.Deployment; +import com.conveyal.datatools.manager.models.EC2InstanceSummary; import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; @@ -62,6 +64,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.conveyal.datatools.manager.controllers.api.ServerController.getIds; import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_OTP_VERSION; import static com.conveyal.datatools.manager.models.Deployment.DEFAULT_R5_VERSION; @@ -75,6 +78,8 @@ public class DeployJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(DeployJob.class); private static final String bundlePrefix = "bundles"; public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; + private static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; + public static final String DEFAULT_AMI_ID = DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); /** * S3 bucket to upload deployment to. If not null, uses {@link OtpServer#s3Bucket}. Otherwise, defaults to * {@link DataManager#feedBucket} @@ -101,16 +106,18 @@ public class DeployJob extends MonitorableJob { private String statusMessage; private int serverCounter = 0; -// private String imageId; private String dateString = DATE_FORMAT.format(new Date()); - List amiNameFilter = Collections.singletonList("ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-????????"); - List amiStateFilter = Collections.singletonList("available"); @JsonProperty public String getDeploymentId () { return deployment.id; } + @JsonProperty + public String getServerId () { + return otpServer.id; + } + public DeployJob(Deployment deployment, String owner, OtpServer otpServer) { // TODO add new job type or get rid of enum in favor of just using class names super(owner, "Deploying " + deployment.name, JobType.DEPLOY_TO_OTP); @@ -128,13 +135,12 @@ public DeployJob(Deployment deployment, String owner, OtpServer otpServer) { // CONNECT TO EC2 // FIXME Should this ec2 client be longlived? ec2 = AmazonEC2Client.builder().build(); -// imageId = ec2.describeImages(new DescribeImagesRequest().withOwners("099720109477").withFilters(new Filter("name", amiNameFilter), new Filter("state", amiStateFilter))).getImages().get(0).getImageId(); } public void jobLogic () { if (otpServer.s3Bucket != null) totalTasks++; // FIXME - if (otpServer.targetGroupArn != null) totalTasks++; + if (otpServer.ec2Info.targetGroupArn != null) totalTasks++; try { deploymentTempFile = File.createTempFile("deployment", ".zip"); } catch (IOException e) { @@ -165,7 +171,7 @@ public void jobLogic () { status.built = true; // Upload to S3, if specifically required by the OTPServer or needed for servers in the target group to fetch. - if (otpServer.s3Bucket != null || otpServer.targetGroupArn != null) { + if (otpServer.s3Bucket != null || otpServer.ec2Info.targetGroupArn != null) { if (!DataManager.useS3) { String message = "Cannot upload deployment to S3. Application not configured for s3 storage."; LOG.error(message); @@ -183,7 +189,7 @@ public void jobLogic () { } // Handle spinning up new EC2 servers for the load balancer's target group. - if (otpServer.targetGroupArn != null) { + if (otpServer.ec2Info.targetGroupArn != null) { if ("true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled"))) { replaceEC2Servers(); // If creating a new server, there is no need to deploy to an existing one. @@ -207,6 +213,9 @@ public void jobLogic () { status.completed = true; } + /** + * Upload to S3 the transit data bundle zip that contains GTFS zip files, OSM data, and config files. + */ private void uploadBundleToS3() throws InterruptedException, AmazonClientException { status.message = "Uploading to s3://" + s3Bucket; status.uploadingS3 = true; @@ -409,25 +418,24 @@ public void jobFinished () { NotifyUsersForSubscriptionJob.createNotification("deployment-updated", deployment.id, message); } - public void replaceEC2Servers() { + + private void replaceEC2Servers() { try { + // Track any previous instances running for the server we're deploying to in order to de-register and + // terminate them later. + List previousInstances = otpServer.retrieveEC2InstanceSummaries(); // First start graph-building instance and wait for graph to successfully build. + status.message = "Starting up graph building EC2 instance"; List instances = startEC2Instances(1); - if (instances.size() > 1) { - // FIXME is this check/shutdown entirely unnecessary? - status.fail("CRITICAL: More than one server initialized for graph building. Cancelling job. Please contact system administrator."); - // Terminate new instances. - // FIXME Should this ec2 client be longlived? - ec2.terminateInstances(new TerminateInstancesRequest(getIds(instances))); - } - // FIXME What if instances list is empty? + status.message = "Waiting for graph build to complete..."; MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob(owner, deployment, instances.get(0), otpServer); monitorInitialServerJob.run(); status.update("Graph build is complete!", 50); // Spin up remaining servers which will download the graph from S3. - int remainingServerCount = otpServer.instanceCount <= 0 ? 0 : otpServer.instanceCount - 1; + int remainingServerCount = otpServer.ec2Info.instanceCount <= 0 ? 0 : otpServer.ec2Info.instanceCount - 1; if (remainingServerCount > 0) { // Spin up remaining EC2 instances. + status.message = String.format("Spinning up remaining %d instance(s).", remainingServerCount); List remainingInstances = startEC2Instances(remainingServerCount); instances.addAll(remainingInstances); // Create new thread pool to monitor server setup so that the servers are monitored in parallel. @@ -443,31 +451,31 @@ public void replaceEC2Servers() { service.awaitTermination(4, TimeUnit.HOURS); } String finalMessage = "Server setup is complete!"; - if (otpServer.instanceIds != null) { + // Get EC2 servers running that are associated with this server. + List instanceIds = previousInstances.stream() + .filter(instance -> "running".equals(instance.state.getName())) + .map(instance -> instance.instanceId) + .collect(Collectors.toList()); + if (instanceIds.size() > 0) { // Deregister old instances from load balancer. (Note: new instances are registered with load balancer in // MonitorServerStatusJob.) - LOG.info("Deregistering instances from load balancer {}", otpServer.instanceIds); - TargetDescription[] targetDescriptions = otpServer.instanceIds - .stream() - .map(id -> new TargetDescription().withId(id)).toArray(TargetDescription[]::new); + LOG.info("De-registering instances from load balancer {}", instanceIds); + TargetDescription[] targetDescriptions = instanceIds.stream() + .map(id -> new TargetDescription().withId(id)) + .toArray(TargetDescription[]::new); DeregisterTargetsRequest deregisterTargetsRequest = new DeregisterTargetsRequest() - .withTargetGroupArn(otpServer.targetGroupArn) - .withTargets(targetDescriptions); - DeregisterTargetsResult deregisterTargetsResult = com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient.builder().build() - .deregisterTargets(deregisterTargetsRequest); - // Terminate old instances. - LOG.info("Terminating instances {}", otpServer.instanceIds); + .withTargetGroupArn(otpServer.ec2Info.targetGroupArn) + .withTargets(targetDescriptions); + AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); + elb.deregisterTargets(deregisterTargetsRequest); try { - TerminateInstancesRequest terminateInstancesRequest = new TerminateInstancesRequest().withInstanceIds(otpServer.instanceIds); - TerminateInstancesResult terminateInstancesResult = ec2.terminateInstances(terminateInstancesRequest); + ServerController.terminateInstances(instanceIds); } catch (AmazonEC2Exception e) { - LOG.warn("Could not terminate EC2 instances {}", otpServer.instanceIds); - finalMessage = String.format("Server setup is complete! (WARNING: Could not terminate previous EC2 instances: %s", otpServer.instanceIds); + LOG.warn("Could not terminate EC2 instances {}", instanceIds); + finalMessage = String.format("Server setup is complete! (WARNING: Could not terminate previous EC2 instances: %s", instanceIds); } } - // Update list of instance IDs with new list. - Persistence.servers.updateField(otpServer.id, "instanceIds", getIds(instances)); - // Job is complete? FIXME Do we need a status check here? + // Job is complete. status.update(false, finalMessage, 100, true); } catch (Exception e) { LOG.error("Could not deploy to EC2 server", e); @@ -475,39 +483,52 @@ public void replaceEC2Servers() { } } + /** + * Start the specified number of EC2 instances based on the {@link OtpServer#ec2Info}. + * @param count number of EC2 instances to start + * @return a list of the instances is returned once the public IP addresses have been assigned + */ private List startEC2Instances(int count) { - String instanceType = otpServer.instanceType == null ? DEFAULT_INSTANCE_TYPE : otpServer.instanceType; + String instanceType = otpServer.ec2Info.instanceType == null ? DEFAULT_INSTANCE_TYPE : otpServer.ec2Info.instanceType; // User data should contain info about: // 1. Downloading GTFS/OSM info (s3) // 2. Time to live until shutdown/termination (for test servers) // 3. Hosting / nginx // FIXME: Allow for r5 servers to be created. - String userData = constructUserData(deployment.r5); + String userData = constructUserData(); // The subnet ID should only change if starting up a server in some other AWS account. This is not // likely to be a requirement. // Define network interface so that a public IP can be associated with server. InstanceNetworkInterfaceSpecification interfaceSpecification = new InstanceNetworkInterfaceSpecification() - .withSubnetId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.subnet")) + .withSubnetId(otpServer.ec2Info.subnetId) .withAssociatePublicIpAddress(true) - .withGroups(DataManager.getConfigPropertyAsText("modules.deployment.ec2.securityGroup")) + .withGroups(otpServer.ec2Info.securityGroupId) .withDeviceIndex(0); - + // If AMI not defined, use the default AMI ID. + String amiId = otpServer.ec2Info.amiId; + if (amiId == null) { + amiId = DEFAULT_AMI_ID; + // Verify that AMI is correctly defined. + if (amiId == null || !ServerController.amiExists(amiId)) { + statusMessage = String.format( + "Default AMI ID (%s) is missing or bad. Should be provided in config at %s", + amiId, + AMI_CONFIG_PATH); + LOG.error(statusMessage); + status.fail(statusMessage); + } + } RunInstancesRequest runInstancesRequest = new RunInstancesRequest() .withNetworkInterfaces(interfaceSpecification) .withInstanceType(instanceType) .withMinCount(count) .withMaxCount(count) - .withImageId(DataManager.getConfigPropertyAsText("modules.deployment.ec2.ami")) - .withKeyName(DataManager.getConfigPropertyAsText("modules.deployment.ec2.keyName")) + .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(otpServer.ec2Info.iamRoleArn)) + .withImageId(amiId) + .withKeyName(otpServer.ec2Info.keyName) // This will have the instance terminate when it is shut down. .withInstanceInitiatedShutdownBehavior("terminate") .withUserData(Base64.encodeBase64String(userData.getBytes())); - // Set IAM instance profile if specified. - if (DataManager.hasConfigProperty("modules.deployment.ec2.arn")) { - IamInstanceProfileSpecification instanceProfile = new IamInstanceProfileSpecification() - .withArn(DataManager.getConfigPropertyAsText("modules.deployment.ec2.arn")); - runInstancesRequest.setIamInstanceProfile(instanceProfile); - } final List instances = ec2.runInstances(runInstancesRequest).getReservation().getInstances(); List instanceIds = getIds(instances); @@ -515,7 +536,6 @@ private List startEC2Instances(int count) { // Wait so that create tags request does not fail because instances not found. try { Waiter waiter = ec2.waiters().instanceStatusOk(); -// ec2.waiters().systemStatusOk() long beginWaiting = System.currentTimeMillis(); waiter.run(new WaiterParameters<>(new DescribeInstanceStatusRequest().withInstanceIds(instanceIds))); LOG.info("Instance status is OK after {} ms", (System.currentTimeMillis() - beginWaiting)); @@ -533,6 +553,9 @@ private List startEC2Instances(int count) { ec2.createTags(new CreateTagsRequest() .withTags(new Tag("Name", serverName)) .withTags(new Tag("projectId", deployment.projectId)) + .withTags(new Tag("deploymentId", deployment.id)) + .withTags(new Tag("jobId", this.jobId)) + .withTags(new Tag("serverId", otpServer.id)) .withResources(instance.getInstanceId()) ); } @@ -543,8 +566,8 @@ private List startEC2Instances(int count) { // been assigned). updatedInstances.clear(); // Check that all of the instances have public IPs. - DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest().withInstanceIds(instanceIds); - List reservations = ec2.describeInstances(describeInstancesRequest).getReservations(); + DescribeInstancesRequest request = new DescribeInstancesRequest().withInstanceIds(instanceIds); + List reservations = ec2.describeInstances(request).getReservations(); for (Reservation reservation : reservations) { for (Instance instance : reservation.getInstances()) { instanceIpAddresses.put(instance.getInstanceId(), instance.getPublicIpAddress()); @@ -552,7 +575,9 @@ private List startEC2Instances(int count) { } } try { - Thread.sleep(10000); + int sleepTimeMillis = 10000; + LOG.info("Waiting {} seconds...", sleepTimeMillis / 1000); + Thread.sleep(sleepTimeMillis); } catch (InterruptedException e) { e.printStackTrace(); } @@ -561,20 +586,20 @@ private List startEC2Instances(int count) { return updatedInstances; } - private List getIds (List instances) { - return instances.stream().map(Instance::getInstanceId).collect(Collectors.toList()); - } - - private String constructUserData(boolean r5) { + /** + * Construct the user data script (as string) that should be provided to the AMI and executed upon EC2 instance + * startup. + */ + private String constructUserData() { // Prefix/name of JAR file (WITHOUT .jar) - String jarName = r5 ? deployment.r5Version : deployment.otpVersion; + String jarName = deployment.r5 ? deployment.r5Version : deployment.otpVersion; if (jarName == null) { // If there is no version specified, use the default (and persist value). - jarName = r5 ? DEFAULT_R5_VERSION : DEFAULT_OTP_VERSION; - Persistence.deployments.updateField(deployment.id, r5 ? "r5Version" : "otpVersion", jarName); + jarName = deployment.r5 ? DEFAULT_R5_VERSION : DEFAULT_OTP_VERSION; + Persistence.deployments.updateField(deployment.id, deployment.r5 ? "r5Version" : "otpVersion", jarName); } - String tripPlanner = r5 ? "r5" : "otp"; - String s3JarBucket = r5 ? "r5-builds" : "opentripplanner-builds"; + String tripPlanner = deployment.r5 ? "r5" : "otp"; + String s3JarBucket = deployment.r5 ? "r5-builds" : "opentripplanner-builds"; String s3JarUrl = String.format("https://%s.s3.amazonaws.com/%s.jar", s3JarBucket, jarName); // TODO Check that jar URL exists? String jarDir = String.format("/opt/%s", tripPlanner); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 23d366a87..2aa392665 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -9,7 +9,6 @@ import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest; -import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsResult; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.models.Deployment; @@ -25,6 +24,10 @@ import java.io.IOException; +/** + * Job that is dispatched during a {@link DeployJob} that spins up EC2 instances. This handles waiting for the server to + * come online and for the OTP application/API to become available. + */ public class MonitorServerStatusJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(MonitorServerStatusJob.class); private final Deployment deployment; @@ -50,7 +53,7 @@ public MonitorServerStatusJob(String owner, Deployment deployment, Instance inst @Override public void jobLogic() { long startTime = System.currentTimeMillis(); - // FIXME use private IP? + // Get OTP URL for instance to check for availability. String otpUrl = String.format("http://%s/otp", instance.getPublicIpAddress()); boolean otpIsRunning = false, routerIsAvailable = false; // Progressively check status of OTP server @@ -82,21 +85,16 @@ public void jobLogic() { routerIsAvailable = checkForSuccessfulRequest(routerUrl, 5); } status.update(String.format("Graph build completed on server %s (%s).", instance.getInstanceId(), routerUrl), 90); - if (otpServer.targetGroupArn != null) { + if (otpServer.ec2Info.targetGroupArn != null) { // After the router is available, the EC2 instance can be registered with the load balancer. // REGISTER INSTANCE WITH LOAD BALANCER AmazonElasticLoadBalancing elbClient = AmazonElasticLoadBalancingClient.builder().build(); RegisterTargetsRequest registerTargetsRequest = new RegisterTargetsRequest() - .withTargetGroupArn(otpServer.targetGroupArn) + .withTargetGroupArn(otpServer.ec2Info.targetGroupArn) .withTargets(new TargetDescription().withId(instance.getInstanceId())); - RegisterTargetsResult registerTargetsResult = elbClient.registerTargets(registerTargetsRequest); -// try { -// elbClient.waiters().targetInService().run(new WaiterParameters<>(new DescribeTargetHealthRequest().withTargetGroupArn(otpServer.targetGroupArn))); -// } catch (Exception e) { -// e.printStackTrace(); -// } + elbClient.registerTargets(registerTargetsRequest); // FIXME how do we know it was successful? - String message = String.format("Server %s successfully registered with load balancer %s", instance.getInstanceId(), otpServer.targetGroupArn); + String message = String.format("Server %s successfully registered with load balancer %s", instance.getInstanceId(), otpServer.ec2Info.targetGroupArn); LOG.info(message); status.update(false, message, 100, true); } else { diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 54b04368b..a927d7cbd 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -1,6 +1,8 @@ package com.conveyal.datatools.manager.models; +import com.amazonaws.services.ec2.model.Filter; import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.controllers.api.DeploymentController; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.StringUtils; import com.conveyal.datatools.manager.utils.json.JsonManager; @@ -29,6 +31,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Locale; @@ -108,6 +111,13 @@ public List retrieveFeedVersions() { return ret; } + /** All of the feed versions used in this deployment, summarized so that the Internet won't break */ + @JsonProperty("ec2Instances") + public List retrieveEC2Instances() { + Filter deploymentFilter = new Filter("tag:deploymentId", Collections.singletonList(id)); + return DeploymentController.fetchEC2InstanceSummaries(deploymentFilter); + } + public void storeFeedVersions(Collection versions) { feedVersionIds = new ArrayList<>(versions.size()); @@ -243,7 +253,8 @@ public Deployment() { // do nothing. } - /** Dump this deployment to the given file + /** + * Dump this deployment to the given output file. * @param output the output file * @param includeOsm should an osm.pbf file be included in the dump? * @param includeOtpConfig should OTP build-config.json and router-config.json be included? @@ -435,9 +446,9 @@ public Rectangle2D retrieveProjectBounds() { /** * Get the deployments currently deployed to a particular server and router combination. */ - public static FindIterable retrieveDeploymentForServerAndRouterId(String server, String routerId) { + public static FindIterable retrieveDeploymentForServerAndRouterId(String serverId, String routerId) { return Persistence.deployments.getMongoCollection().find(and( - eq("deployedTo", server), + eq("deployedTo", serverId), eq("routerId", routerId) )); } diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java new file mode 100644 index 000000000..fc5d406c8 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -0,0 +1,36 @@ +package com.conveyal.datatools.manager.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.io.Serializable; +import java.util.List; + +/** + * Contains the fields specific to starting up new EC2 servers for an ELB target group. If null, at least one internal + * URLs must be provided. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class EC2Info implements Serializable { + private static final long serialVersionUID = 1L; + /** Empty constructor for serialization. */ + public EC2Info () {} + /** + * The AWS-style instance type (e.g., t2.medium) to use for new EC2 machines. Defaults to + * {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} if null during deployment. + */ + public String instanceType; + /** Number of instances to spin up and add to target group. If zero, defaults to 1. */ + public int instanceCount; + /** The subnet ID associated with the target group. */ + public String subnetId; + /** The security group ID associated with the target group. */ + public String securityGroupId; + /** The Amazon machine image (AMI) to be used for the OTP EC2 machines. */ + public String amiId; + /** The IAM role ARN that the OTP EC2 server should assume. */ + public String iamRoleArn; + /** The AWS key file (.pem) that should be used to set up OTP EC2 servers (gives a way for admins to SSH into machine). */ + public String keyName; + /** The target group to deploy new EC2 instances to. */ + public String targetGroupArn; +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java new file mode 100644 index 000000000..717605c58 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java @@ -0,0 +1,56 @@ +package com.conveyal.datatools.manager.models; + +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.InstanceState; +import com.amazonaws.services.ec2.model.Tag; + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + +/** + * Summarizes information derived from an EC2 instance for consumption by a user interface. + */ +public class EC2InstanceSummary implements Serializable { + private static final long serialVersionUID = 1L; + public final String privateIpAddress; + public final String publicIpAddress; + public final String publicDnsName; + public final String instanceType; + public final String instanceId; + public final String imageId; + public final String projectId; + public final String deploymentId; + public final String name; + public final InstanceState state; + public final String availabilityZone; + public final Date launchTime; + public final String stateTransitionReason; + + + public EC2InstanceSummary (Instance ec2Instance) { + publicIpAddress = ec2Instance.getPublicIpAddress(); + privateIpAddress = ec2Instance.getPrivateIpAddress(); + publicDnsName = ec2Instance.getPublicDnsName(); + instanceType = ec2Instance.getInstanceType(); + instanceId = ec2Instance.getInstanceId(); + imageId = ec2Instance.getImageId(); + List tags = ec2Instance.getTags(); + // Set project and deployment ID if they exist. + String projectId = null; + String deploymentId = null; + String name = null; + for (Tag tag : tags) { + if (tag.getKey().equals("projectId")) projectId = tag.getValue(); + if (tag.getKey().equals("deploymentId")) deploymentId = tag.getValue(); + if (tag.getKey().equals("Name")) name = tag.getValue(); + } + this.projectId = projectId; + this.deploymentId = deploymentId; + this.name = name; + state = ec2Instance.getState(); + availabilityZone = ec2Instance.getPlacement().getAvailabilityZone(); + launchTime = ec2Instance.getLaunchTime(); + stateTransitionReason = ec2Instance.getStateTransitionReason(); + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java index d24dc8946..de50a0c6c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java +++ b/src/main/java/com/conveyal/datatools/manager/models/OtpServer.java @@ -1,29 +1,65 @@ package com.conveyal.datatools.manager.models; +import com.amazonaws.services.ec2.model.Filter; +import com.amazonaws.services.ec2.model.Instance; +import com.conveyal.datatools.manager.controllers.api.DeploymentController; import com.conveyal.datatools.manager.persistence.Persistence; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collections; import java.util.List; /** + * An OtpServer represents a deployment target for deploying transit and OSM data to. This can take the shape of a number + * of things: + * 1. Simply writing a data bundle to S3. + * 2. Deploying to an internal URL for a build graph over wire request. + * 3. Spinning up an EC2 instance to build the graph, write it to S3, and have a collection of instances start up, become + * part of an Elastic Load Balancer (ELB) target group, and download/read in the OTP graph. + * read in that graph. + * 4. Spinning up an EC2 instance to only build the OTP graph and write it to S3 (dependent on {@link Deployment#buildGraphOnly} + * value). + * * Created by landon on 5/20/16. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class OtpServer extends Model { private static final long serialVersionUID = 1L; public String name; + /** URL to direct build graph over wire requests to (if not using ELB target group). */ public List internalUrl; - public List instanceIds; - public String instanceType; - public int instanceCount; + /** Optional project to associate this server with (server can also be made available to entire application). */ public String projectId; - public String targetGroupArn; + /** Contains all of the information needed to commission EC2 instances for an AWS Elastic Load Balancer (ELB) target group. */ + public EC2Info ec2Info; + /** + * URL location of the publicly-available user interface asssociated with either the {@link #internalUrl} or the + * load balancer/target group. + */ public String publicUrl; + /** Whether deploying to this server is limited to admins only. */ public boolean admin; + /** S3 bucket name to upload deployment artifacts to (e.g., Graph.obj and/or transit + OSM data). */ public String s3Bucket; /** Empty constructor for serialization. */ public OtpServer () {} + /** The EC2 instances that are associated with this serverId. */ + @JsonProperty("ec2Instances") + public List retrieveEC2InstanceSummaries() { + // Prevent calling EC2 method on servers that do not have EC2 info defined because this is a JSON property. + if (ec2Info == null) return Collections.EMPTY_LIST; + Filter serverFilter = new Filter("tag:serverId", Collections.singletonList(id)); + return DeploymentController.fetchEC2InstanceSummaries(serverFilter); + } + + public List retrieveEC2Instances() { + Filter serverFilter = new Filter("tag:serverId", Collections.singletonList(id)); + return DeploymentController.fetchEC2Instances(serverFilter); + } + @JsonProperty("organizationId") public String organizationId() { Project project = parentProject(); From 9b957dd861ac317cef3640d778feb2803735138a Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 10 Sep 2019 10:14:23 -0400 Subject: [PATCH 25/42] refactor(deploy): tweak deployJob for NPE fix and fix server delete --- .../manager/controllers/api/ServerController.java | 8 +++++--- .../com/conveyal/datatools/manager/jobs/DeployJob.java | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 11b109fe2..83cc91359 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -94,9 +94,11 @@ private static OtpServer checkServerPermissions(Request req, Response res) { private static OtpServer deleteServer(Request req, Response res) { OtpServer server = checkServerPermissions(req, res); - List instances = server.retrieveEC2Instances(); - if (instances.size() > 0) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Cannot delete server with active EC2 instances: " + getIds(instances)); + List activeInstances = server.retrieveEC2Instances().stream() + .filter(instance -> "running".equals(instance.getState().getName())) + .collect(Collectors.toList()); + if (activeInstances.size() > 0) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Cannot delete server with active EC2 instances: " + getIds(activeInstances)); } server.delete(); return server; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index dad7b3ff9..f21125526 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -139,8 +139,7 @@ public DeployJob(Deployment deployment, String owner, OtpServer otpServer) { public void jobLogic () { if (otpServer.s3Bucket != null) totalTasks++; - // FIXME - if (otpServer.ec2Info.targetGroupArn != null) totalTasks++; + if (otpServer.ec2Info != null) totalTasks++; try { deploymentTempFile = File.createTempFile("deployment", ".zip"); } catch (IOException e) { @@ -171,7 +170,7 @@ public void jobLogic () { status.built = true; // Upload to S3, if specifically required by the OTPServer or needed for servers in the target group to fetch. - if (otpServer.s3Bucket != null || otpServer.ec2Info.targetGroupArn != null) { + if (otpServer.s3Bucket != null || otpServer.ec2Info != null) { if (!DataManager.useS3) { String message = "Cannot upload deployment to S3. Application not configured for s3 storage."; LOG.error(message); @@ -189,7 +188,7 @@ public void jobLogic () { } // Handle spinning up new EC2 servers for the load balancer's target group. - if (otpServer.ec2Info.targetGroupArn != null) { + if (otpServer.ec2Info != null) { if ("true".equals(DataManager.getConfigPropertyAsText("modules.deployment.ec2.enabled"))) { replaceEC2Servers(); // If creating a new server, there is no need to deploy to an existing one. From bf0f1bc10db906da79e64f4fb366ff626449a774 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 10 Sep 2019 10:48:13 -0400 Subject: [PATCH 26/42] ci(config): update server.yml.tmp for e2e --- configurations/default/server.yml.tmp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index 16249fe4d..422edb1ee 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -1,7 +1,12 @@ application: + title: Data Tools + logo: https://d2tyb7byn1fef9.cloudfront.net/ibi_group-128x128.png + logo_large: https://d2tyb7byn1fef9.cloudfront.net/ibi_group_black-512x512.png assets_bucket: bucket-name public_url: http://localhost:9966 notifications_enabled: false + docs_url: http://conveyal-data-tools.readthedocs.org + support_email: support@ibigroup.com port: 4000 data: gtfs: /tmp @@ -14,11 +19,12 @@ modules: deployment: enabled: true editor: - enabled: false + enabled: true deployment: - enabled: false + enabled: true ec2: enabled: false + default_ami: ami-your-ami-id user_admin: enabled: true # Enable GTFS+ module for testing purposes From ade0b40ef06362acbb7d4f04b9c0dbc410e5888c Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 10 Sep 2019 13:47:35 -0400 Subject: [PATCH 27/42] refactor(EC2InstanceSummary): add empty constructor for serialization --- .../manager/models/EC2InstanceSummary.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java index 717605c58..d27ad9005 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java @@ -13,20 +13,22 @@ */ public class EC2InstanceSummary implements Serializable { private static final long serialVersionUID = 1L; - public final String privateIpAddress; - public final String publicIpAddress; - public final String publicDnsName; - public final String instanceType; - public final String instanceId; - public final String imageId; - public final String projectId; - public final String deploymentId; - public final String name; - public final InstanceState state; - public final String availabilityZone; - public final Date launchTime; - public final String stateTransitionReason; + public String privateIpAddress; + public String publicIpAddress; + public String publicDnsName; + public String instanceType; + public String instanceId; + public String imageId; + public String projectId; + public String deploymentId; + public String name; + public InstanceState state; + public String availabilityZone; + public Date launchTime; + public String stateTransitionReason; + /** Empty constructor for serialization */ + public EC2InstanceSummary () { } public EC2InstanceSummary (Instance ec2Instance) { publicIpAddress = ec2Instance.getPublicIpAddress(); From c273350ea54c0dcb0c4c8abf2f45b2597592d4c0 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 12 Sep 2019 11:46:58 -0400 Subject: [PATCH 28/42] test(.gitignore): don't ignore test config --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 66c303089..b04d422d8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ deploy/ # Configurations configurations/* !configurations/default +!configurations/test # Secret config files .env From 3493570646c16daa43443c8972b24e574334bb04 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 12 Sep 2019 11:49:25 -0400 Subject: [PATCH 29/42] test(mtc): fix broken MTC feed merge test with new test config --- configurations/default/server.yml.tmp | 5 -- configurations/test/env.yml.tmp | 19 +++++++ configurations/test/server.yml.tmp | 50 +++++++++++++++++++ .../com/conveyal/datatools/DatatoolsTest.java | 2 +- 4 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 configurations/test/env.yml.tmp create mode 100644 configurations/test/server.yml.tmp diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index cd5c17655..62df3dd69 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -17,8 +17,6 @@ application: modules: enterprise: enabled: false - deployment: - enabled: true editor: enabled: true deployment: @@ -28,9 +26,6 @@ modules: default_ami: ami-your-ami-id user_admin: enabled: true - # Enable GTFS+ module for testing purposes - gtfsplus: - enabled: true gtfsapi: enabled: true load_on_fetch: false diff --git a/configurations/test/env.yml.tmp b/configurations/test/env.yml.tmp new file mode 100644 index 000000000..eb5769962 --- /dev/null +++ b/configurations/test/env.yml.tmp @@ -0,0 +1,19 @@ +# This client ID refers to the UI client in Auth0. +AUTH0_CLIENT_ID: your-auth0-client-id +AUTH0_DOMAIN: your-auth0-domain +# Note: One of AUTH0_SECRET or AUTH0_PUBLIC_KEY should be used depending on the signing algorithm set on the client. +# It seems that newer Auth0 accounts (2017 and later) might default to RS256 (public key). +AUTH0_SECRET: your-auth0-secret # uses HS256 signing algorithm +# AUTH0_PUBLIC_KEY: /path/to/auth0.pem # uses RS256 signing algorithm +# This client/secret pair refer to a machine-to-machine Auth0 application used to access the Management API. +AUTH0_API_CLIENT: your-api-client-id +AUTH0_API_SECRET: your-api-secret-id +DISABLE_AUTH: false +OSM_VEX: http://localhost:1000 +SPARKPOST_KEY: your-sparkpost-key +SPARKPOST_EMAIL: email@example.com +GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue +# GTFS_DATABASE_USER: +# GTFS_DATABASE_PASSWORD: +#MONGO_URI: mongodb://mongo-host:27017 +MONGO_DB_NAME: catalogue diff --git a/configurations/test/server.yml.tmp b/configurations/test/server.yml.tmp new file mode 100644 index 000000000..be50409b8 --- /dev/null +++ b/configurations/test/server.yml.tmp @@ -0,0 +1,50 @@ +application: + title: Data Tools + logo: https://d2tyb7byn1fef9.cloudfront.net/ibi_group-128x128.png + logo_large: https://d2tyb7byn1fef9.cloudfront.net/ibi_group_black-512x512.png + client_assets_url: https://example.com + shortcut_icon_url: https://d2tyb7byn1fef9.cloudfront.net/ibi-logo-original%402x.png + public_url: http://localhost:9966 + notifications_enabled: false + docs_url: http://conveyal-data-tools.readthedocs.org + support_email: support@ibigroup.com + port: 4000 + data: + gtfs: /tmp + use_s3_storage: false + s3_region: us-east-1 + gtfs_s3_bucket: bucket-name +modules: + enterprise: + enabled: false + editor: + enabled: true + deployment: + enabled: true + ec2: + enabled: false + default_ami: ami-your-ami-id + user_admin: + enabled: true + # Enable GTFS+ module for testing purposes + gtfsplus: + enabled: true + gtfsapi: + enabled: true + load_on_fetch: false + # use_extension: mtc + # update_frequency: 30 # in seconds +extensions: + # Enable MTC extension so MTC-specific feed merge tests + mtc: + enabled: true + rtd_api: http://localhost:9876/ + s3_bucket: bucket-name + s3_prefix: waiting/ + s3_download_prefix: waiting/ + transitland: + enabled: true + api: https://transit.land/api/v1/feeds + transitfeeds: + enabled: true + api: http://api.transitfeeds.com/v1/getFeeds diff --git a/src/test/java/com/conveyal/datatools/DatatoolsTest.java b/src/test/java/com/conveyal/datatools/DatatoolsTest.java index 0f39ff761..7ce63b3fe 100644 --- a/src/test/java/com/conveyal/datatools/DatatoolsTest.java +++ b/src/test/java/com/conveyal/datatools/DatatoolsTest.java @@ -43,7 +43,7 @@ public static void setUp() throws RuntimeException, IOException { // Travis CI, these files will automatically be setup. String[] args = getBooleanEnvVar("RUN_E2E") ? new String[] { "configurations/default/env.yml", "configurations/default/server.yml" } - : new String[] { "configurations/default/env.yml.tmp", "configurations/default/server.yml.tmp" }; + : new String[] { "configurations/test/env.yml.tmp", "configurations/test/server.yml.tmp" }; // fail this test and others if the above files do not exist for (String arg : args) { From 4992b329dda2b479643cf8002f311b00f300b1ba Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 12 Sep 2019 14:17:15 -0400 Subject: [PATCH 30/42] refactor(ServerController): isolate jackson parse to utility method --- .../controllers/api/ServerController.java | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 83cc91359..82bfeeb1a 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -37,7 +37,6 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.json.JsonManager; import com.fasterxml.jackson.databind.ObjectMapper; -import org.bson.Document; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,26 +134,33 @@ public static TerminateInstancesResult terminateInstances(Collection ins */ private static OtpServer createServer(Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); - Document newServerFields = Document.parse(req.body()); - String projectId = newServerFields.getString("projectId"); - String organizationId = newServerFields.getString("organizationId"); + OtpServer newServer = getServerFromRequestBody(req); // If server has no project ID specified, user must be an application admin to create it. Otherwise, they must // be a project admin. - boolean allowedToCreate = projectId == null + boolean allowedToCreate = newServer.projectId == null ? userProfile.canAdministerApplication() - : userProfile.canAdministerProject(projectId, organizationId); + : userProfile.canAdministerProject(newServer.projectId, newServer.organizationId()); if (allowedToCreate) { try { - OtpServer newServer = mapper.readValue(newServerFields.toJson(), OtpServer.class); validateFields(req, newServer); - Persistence.servers.create(newServer); - return newServer; - } catch (IOException e) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Error parsing OTP server JSON.", e); - return null; + } catch (Exception e) { + if (e instanceof HaltException) throw e; + else logMessageAndHalt(req, 400, "Error encountered while validating server field", e); } + Persistence.servers.create(newServer); + return newServer; } else { - logMessageAndHalt(req, 403, "Not authorized to create a server for project " + projectId); + logMessageAndHalt(req, 403, "Not authorized to create a server for project " + newServer.projectId); + return null; + } + } + + /** Utility method to parse OtpServer object from Spark request body. */ + private static OtpServer getServerFromRequestBody(Request req) { + try { + return mapper.readValue(req.body(), OtpServer.class); + } catch (IOException e) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Error parsing OTP server JSON.", e); return null; } } @@ -180,12 +186,7 @@ private static List fetchServers (Request req, Response res) { */ private static OtpServer updateServer(Request req, Response res) { OtpServer serverToUpdate = checkServerPermissions(req, res); - OtpServer updatedServer = null; - try { - updatedServer = mapper.readValue(req.body(), OtpServer.class); - } catch (IOException e) { - e.printStackTrace(); - } + OtpServer updatedServer = getServerFromRequestBody(req); Auth0UserProfile user = req.attribute("user"); if ((serverToUpdate.admin || serverToUpdate.projectId == null) && !user.canAdministerApplication()) { logMessageAndHalt(req, HttpStatus.UNAUTHORIZED_401, "User cannot modify admin-only or application-wide server."); From 3441fa22ddbf49d93ceada5ccb778539c7b5a7ef Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 12 Sep 2019 14:19:38 -0400 Subject: [PATCH 31/42] refactor(ServerController): surround validation method calls in try/catch --- .../controllers/api/ServerController.java | 86 ++++++++++--------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 82bfeeb1a..01c4fc831 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -141,12 +141,7 @@ private static OtpServer createServer(Request req, Response res) { ? userProfile.canAdministerApplication() : userProfile.canAdministerProject(newServer.projectId, newServer.organizationId()); if (allowedToCreate) { - try { - validateFields(req, newServer); - } catch (Exception e) { - if (e instanceof HaltException) throw e; - else logMessageAndHalt(req, 400, "Error encountered while validating server field", e); - } + validateFields(req, newServer); Persistence.servers.create(newServer); return newServer; } else { @@ -201,44 +196,51 @@ private static OtpServer updateServer(Request req, Response res) { * removing problematic date fields. */ private static void validateFields(Request req, OtpServer server) throws HaltException { - // Check that projectId is valid. - if (server.projectId != null) { - Project project = Persistence.projects.getById(server.projectId); - if (project == null) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Must specify valid project ID."); - } - // If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to - // AWS. (e.g., target group ARN and instance type). - if (server.ec2Info != null) { - validateTargetGroup(server.ec2Info.targetGroupArn, req); - validateInstanceType(server.ec2Info.instanceType, req); - validateSubnetId(server.ec2Info.subnetId, req); - validateSecurityGroupId(server.ec2Info.securityGroupId, req); - validateIamRoleArn(server.ec2Info.iamRoleArn, req); - validateKeyName(server.ec2Info.keyName, req); - validateAmiId(server.ec2Info.amiId, req); - if (server.ec2Info.instanceCount < 0) server.ec2Info.instanceCount = 0; - } - // Server must have name. - if (isEmpty(server.name)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must have valid name."); - // Server must have an internal URL (for build graph over wire) or an s3 bucket (for auto deploy ec2). - if (isEmpty(server.s3Bucket)) { - if (server.internalUrl == null || server.internalUrl.size() == 0) { - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must contain either internal URL(s) or s3 bucket name."); + try { + // Check that projectId is valid. + if (server.projectId != null) { + Project project = Persistence.projects.getById(server.projectId); + if (project == null) + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Must specify valid project ID."); } - } else { - // Verify that application has permission to write to/delete from S3 bucket. We're following the recommended - // approach from https://stackoverflow.com/a/17284647/915811, but perhaps there is a way to do this - // effectively without incurring AWS costs (although writing/deleting an empty file to S3 is probably - // miniscule). - String key = UUID.randomUUID().toString(); - try { - FeedStore.s3Client.putObject(server.s3Bucket, key, File.createTempFile("test", ".zip")); - FeedStore.s3Client.deleteObject(server.s3Bucket, key); - } catch (IOException | AmazonS3Exception e) { - String message = "Cannot write to specified S3 bucket " + server.s3Bucket; - LOG.error(message, e); - logMessageAndHalt(req, 400, message, e); + // If a server's ec2 info object is not null, it must pass a few validation checks on various fields related to + // AWS. (e.g., target group ARN and instance type). + if (server.ec2Info != null) { + validateTargetGroup(server.ec2Info.targetGroupArn, req); + validateInstanceType(server.ec2Info.instanceType, req); + validateSubnetId(server.ec2Info.subnetId, req); + validateSecurityGroupId(server.ec2Info.securityGroupId, req); + validateIamRoleArn(server.ec2Info.iamRoleArn, req); + validateKeyName(server.ec2Info.keyName, req); + validateAmiId(server.ec2Info.amiId, req); + if (server.ec2Info.instanceCount < 0) server.ec2Info.instanceCount = 0; + } + // Server must have name. + if (isEmpty(server.name)) + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must have valid name."); + // Server must have an internal URL (for build graph over wire) or an s3 bucket (for auto deploy ec2). + if (isEmpty(server.s3Bucket)) { + if (server.internalUrl == null || server.internalUrl.size() == 0) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must contain either internal URL(s) or s3 bucket name."); + } + } else { + // Verify that application has permission to write to/delete from S3 bucket. We're following the recommended + // approach from https://stackoverflow.com/a/17284647/915811, but perhaps there is a way to do this + // effectively without incurring AWS costs (although writing/deleting an empty file to S3 is probably + // miniscule). + String key = UUID.randomUUID().toString(); + try { + FeedStore.s3Client.putObject(server.s3Bucket, key, File.createTempFile("test", ".zip")); + FeedStore.s3Client.deleteObject(server.s3Bucket, key); + } catch (IOException | AmazonS3Exception e) { + String message = "Cannot write to specified S3 bucket " + server.s3Bucket; + LOG.error(message, e); + logMessageAndHalt(req, 400, message, e); + } } + } catch (Exception e) { + if (e instanceof HaltException) throw e; + else logMessageAndHalt(req, 400, "Error encountered while validating server field", e); } } From a36a7d9643d223cc3357c7e5b0136e08ffd0f154 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 20 Sep 2019 14:40:52 -0400 Subject: [PATCH 32/42] refactor(deploy-to-ec2): address PR comments --- .../common/status/MonitorableJob.java | 10 +- .../manager/auth/Auth0Connection.java | 4 + .../controllers/api/DeploymentController.java | 56 ++- .../api/OrganizationController.java | 6 +- .../controllers/api/ServerController.java | 133 +++++-- .../controllers/api/StatusController.java | 17 + .../datatools/manager/jobs/DeployJob.java | 351 +++++++++++++----- .../manager/jobs/MonitorServerStatusJob.java | 148 ++++++-- .../datatools/manager/models/Deployment.java | 13 +- .../datatools/manager/models/EC2Info.java | 4 +- .../manager/models/EC2InstanceSummary.java | 8 +- .../manager/models/Organization.java | 4 +- .../manager/persistence/FeedStore.java | 2 +- .../manager/persistence/Persistence.java | 1 + .../manager/persistence/TypedPersistence.java | 2 +- 15 files changed, 583 insertions(+), 176 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java index 673008be7..7b9516515 100644 --- a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java +++ b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java @@ -7,6 +7,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.Serializable; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -18,7 +19,8 @@ /** * Created by landon on 6/13/16. */ -public abstract class MonitorableJob implements Runnable { +public abstract class MonitorableJob implements Runnable, Serializable { + private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(MonitorableJob.class); public final String owner; @@ -129,7 +131,6 @@ public void run () { boolean parentJobErrored = false; boolean subTaskErrored = false; String cancelMessage = ""; - long startTimeNanos = System.nanoTime(); try { // First execute the core logic of the specific MonitorableJob subclass jobLogic(); @@ -188,8 +189,7 @@ public void run () { LOG.error("Job failed", ex); status.update(true, ex.getMessage(), 100, true); } - status.startTime = TimeUnit.NANOSECONDS.toMillis(startTimeNanos); - status.duration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNanos); + status.duration = System.currentTimeMillis() - status.startTime; LOG.info("{} {} {} in {} ms", type, jobId, status.error ? "errored" : "completed", status.duration); } @@ -243,7 +243,7 @@ public static class Status { /** How much of task is complete? */ public double percentComplete; - public long startTime; + public long startTime = System.currentTimeMillis(); public long duration; // When was the job initialized? diff --git a/src/main/java/com/conveyal/datatools/manager/auth/Auth0Connection.java b/src/main/java/com/conveyal/datatools/manager/auth/Auth0Connection.java index 80c5942ce..6e8ac8787 100644 --- a/src/main/java/com/conveyal/datatools/manager/auth/Auth0Connection.java +++ b/src/main/java/com/conveyal/datatools/manager/auth/Auth0Connection.java @@ -1,5 +1,6 @@ package com.conveyal.datatools.manager.auth; +import com.auth0.jwt.JWTExpiredException; import com.auth0.jwt.JWTVerifier; import com.auth0.jwt.pem.PemReader; import com.conveyal.datatools.manager.DataManager; @@ -90,6 +91,9 @@ public static void checkUser(Request req) { // The user attribute is used on the server side to check user permissions and does not have all of the // fields that the raw Auth0 profile string does. req.attribute("user", profile); + } catch (JWTExpiredException e) { + LOG.warn("JWT token has expired for user."); + logMessageAndHalt(req, 401, "User's authentication token has expired. Please re-login."); } catch (Exception e) { LOG.warn("Login failed to verify with our authorization provider.", e); logMessageAndHalt(req, 401, "Could not verify user's token"); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index d3bc5ed11..d3921f207 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -7,6 +7,7 @@ import com.amazonaws.services.ec2.model.Filter; import com.amazonaws.services.ec2.model.Instance; import com.amazonaws.services.ec2.model.Reservation; +import com.amazonaws.services.s3.AmazonS3URI; import com.conveyal.datatools.common.utils.SparkUtils; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.auth.Auth0UserProfile; @@ -18,8 +19,10 @@ import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.json.JsonManager; +import com.mongodb.client.FindIterable; import org.bson.Document; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; @@ -38,6 +41,7 @@ import java.util.Map; import java.util.stream.Collectors; +import static com.conveyal.datatools.common.utils.S3Utils.downloadFromS3; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static spark.Spark.delete; import static spark.Spark.get; @@ -84,6 +88,46 @@ private static Deployment deleteDeployment (Request req, Response res) { return deployment; } + /** + * HTTP endpoint for downloading a build artifact (e.g., otp build log or Graph.obj) from S3. + */ + private static String downloadBuildArtifact (Request req, Response res) { + Deployment deployment = checkDeploymentPermissions(req, res); + DeployJob.DeploySummary summaryToDownload = null; + String uriString = null; + // If a jobId query param is provided, find the matching job summary. + String jobId = req.queryParams("jobId"); + if (jobId != null) { + for (DeployJob.DeploySummary summary : deployment.deployJobSummaries) { + if (summary.jobId.equals(jobId)) { + summaryToDownload = summary; + break; + } + } + } else { + summaryToDownload = deployment.latest(); + } + if (summaryToDownload == null) { + // Try to construct the URI string + OtpServer server = Persistence.servers.getById(deployment.deployedTo); + if (server == null) { + uriString = String.format("s3://%s/bundles/%s/%s/%s", "S3_BUCKET", deployment.projectId, deployment.id, jobId); + logMessageAndHalt(req, 400, "Cannot construct URI for build artifact. " + uriString); + return null; + } + uriString = String.format("s3://%s/bundles/%s/%s/%s", server.s3Bucket, deployment.projectId, deployment.id, jobId); + LOG.warn("Could not find deploy summary for job. Attempting to use {}", uriString); + } else { + uriString = summaryToDownload.buildArtifactsFolder; + } + AmazonS3URI uri = new AmazonS3URI(uriString); + String filename = req.queryParams("filename"); + if (filename == null) { + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Must provide filename query param for build artifact."); + } + return downloadFromS3(FeedStore.s3Client, uri.getBucket(), String.join("/", uri.getKey(), filename), false, res); + } + /** * Download all of the GTFS files in the feed. * @@ -266,6 +310,9 @@ public static List fetchEC2InstanceSummaries(Filter... filte return fetchEC2Instances(filters).stream().map(EC2InstanceSummary::new).collect(Collectors.toList()); } + /** + * Fetch EC2 instances from AWS that match the provided set of filters (e.g., tags, instance ID, or other properties). + */ public static List fetchEC2Instances(Filter... filters) { List instances = new ArrayList<>(); DescribeInstancesRequest request = new DescribeInstancesRequest().withFilters(filters); @@ -291,7 +338,10 @@ private static String deploy (Request req, Response res) { logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); // Get server by ID OtpServer otpServer = Persistence.servers.getById(target); - if (otpServer == null) logMessageAndHalt(req, 400, "Must provide valid OTP server target ID."); + if (otpServer == null) { + logMessageAndHalt(req, 400, "Must provide valid OTP server target ID."); + return null; + } // Check that permissions of user allow them to deploy to target. boolean isProjectAdmin = userProfile.canAdministerProject(deployment.projectId, deployment.organizationId()); @@ -350,7 +400,11 @@ public static void register (String apiPrefix) { }), json::write); options(apiPrefix + "secure/deployments", (q, s) -> ""); get(apiPrefix + "secure/deployments/:id/download", DeploymentController::downloadDeployment); + get(apiPrefix + "secure/deployments/:id/artifact", DeploymentController::downloadBuildArtifact); get(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::fetchEC2InstanceSummaries, json::write); + // TODO: In the future, we may have need for terminating a single EC2 instance. For now, an admin using the AWS + // console should suffice. +// delete(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::terminateEC2Instance, json::write); get(apiPrefix + "secure/deployments/:id", DeploymentController::getDeployment, json::write); delete(apiPrefix + "secure/deployments/:id", DeploymentController::deleteDeployment, json::write); get(apiPrefix + "secure/deployments", DeploymentController::getAllDeployments, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/OrganizationController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/OrganizationController.java index ddf712093..046c0be32 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/OrganizationController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/OrganizationController.java @@ -68,8 +68,8 @@ public static Organization createOrganization (Request req, Response res) { public static Organization updateOrganization (Request req, Response res) throws IOException { String organizationId = req.params("id"); - requestOrganizationById(req); - Organization organization = Persistence.organizations.update(organizationId, req.body()); + Organization updatedOrganization = requestOrganizationById(req); + Persistence.organizations.replace(organizationId, updatedOrganization); // FIXME: Add back in hook after organization is updated. // See https://github.com/catalogueglobal/datatools-server/issues/111 @@ -101,7 +101,7 @@ public static Organization updateOrganization (Request req, Response res) throws // p.save(); // } - return organization; + return updatedOrganization; } public static Organization deleteOrganization (Request req, Response res) { diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 01c4fc831..c5afa39ff 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -27,6 +27,7 @@ import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClientBuilder; import com.amazonaws.services.identitymanagement.model.InstanceProfile; import com.amazonaws.services.identitymanagement.model.ListInstanceProfilesResult; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.Deployment; @@ -46,6 +47,7 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -76,7 +78,7 @@ public class ServerController { * Gets the server specified by the request's id parameter and ensure that user has access to the * deployment. If the user does not have permission the Spark request is halted with an error. */ - private static OtpServer checkServerPermissions(Request req, Response res) { + private static OtpServer getServerWithPermissions(Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); String serverId = req.params("id"); OtpServer server = Persistence.servers.getById(serverId); @@ -91,8 +93,10 @@ private static OtpServer checkServerPermissions(Request req, Response res) { return server; } + /** HTTP endpoint for deleting an {@link OtpServer}. */ private static OtpServer deleteServer(Request req, Response res) { - OtpServer server = checkServerPermissions(req, res); + OtpServer server = getServerWithPermissions(req, res); + // Ensure that there are no active EC2 instances associated with server. Halt deletion if so. List activeInstances = server.retrieveEC2Instances().stream() .filter(instance -> "running".equals(instance.getState().getName())) .collect(Collectors.toList()); @@ -104,8 +108,8 @@ private static OtpServer deleteServer(Request req, Response res) { } /** HTTP method for terminating EC2 instances associated with an ELB OTP server. */ - private static OtpServer terminateEC2Instances(Request req, Response res) { - OtpServer server = checkServerPermissions(req, res); + private static OtpServer terminateEC2InstancesForServer(Request req, Response res) { + OtpServer server = getServerWithPermissions(req, res); List instances = server.retrieveEC2Instances(); List ids = getIds(instances); terminateInstances(ids); @@ -122,12 +126,21 @@ public static List getIds (List instances) { return instances.stream().map(Instance::getInstanceId).collect(Collectors.toList()); } + /** Terminate the list of EC2 instance IDs. */ public static TerminateInstancesResult terminateInstances(Collection instanceIds) throws AmazonEC2Exception { + if (instanceIds.size() == 0) { + LOG.warn("No instance IDs provided in list. Skipping termination request."); + } LOG.info("Terminating EC2 instances {}", instanceIds); TerminateInstancesRequest request = new TerminateInstancesRequest().withInstanceIds(instanceIds); return ec2.terminateInstances(request); } + /** Convenience method to override {@link #terminateInstances(Collection)}. */ + public static TerminateInstancesResult terminateInstances(String... instanceIds) throws AmazonEC2Exception { + return terminateInstances(Arrays.asList(instanceIds)); + } + /** * Create a new server for the project. All feed sources with a valid latest version are added to the new * deployment. @@ -180,7 +193,7 @@ private static List fetchServers (Request req, Response res) { * Update a single OTP server. */ private static OtpServer updateServer(Request req, Response res) { - OtpServer serverToUpdate = checkServerPermissions(req, res); + OtpServer serverToUpdate = getServerWithPermissions(req, res); OtpServer updatedServer = getServerFromRequestBody(req); Auth0UserProfile user = req.attribute("user"); if ((serverToUpdate.admin || serverToUpdate.projectId == null) && !user.canAdministerApplication()) { @@ -210,7 +223,7 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc validateInstanceType(server.ec2Info.instanceType, req); validateSubnetId(server.ec2Info.subnetId, req); validateSecurityGroupId(server.ec2Info.securityGroupId, req); - validateIamRoleArn(server.ec2Info.iamRoleArn, req); + validateIamInstanceProfileArn(server.ec2Info.iamInstanceProfileArn, req); validateKeyName(server.ec2Info.keyName, req); validateAmiId(server.ec2Info.amiId, req); if (server.ec2Info.instanceCount < 0) server.ec2Info.instanceCount = 0; @@ -224,19 +237,7 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Server must contain either internal URL(s) or s3 bucket name."); } } else { - // Verify that application has permission to write to/delete from S3 bucket. We're following the recommended - // approach from https://stackoverflow.com/a/17284647/915811, but perhaps there is a way to do this - // effectively without incurring AWS costs (although writing/deleting an empty file to S3 is probably - // miniscule). - String key = UUID.randomUUID().toString(); - try { - FeedStore.s3Client.putObject(server.s3Bucket, key, File.createTempFile("test", ".zip")); - FeedStore.s3Client.deleteObject(server.s3Bucket, key); - } catch (IOException | AmazonS3Exception e) { - String message = "Cannot write to specified S3 bucket " + server.s3Bucket; - LOG.error(message, e); - logMessageAndHalt(req, 400, message, e); - } + verifyS3WritePermissions(server, req); } } catch (Exception e) { if (e instanceof HaltException) throw e; @@ -244,12 +245,72 @@ private static void validateFields(Request req, OtpServer server) throws HaltExc } } + /** + * Verify that application has permission to write to/delete from S3 bucket. We're following the recommended + * approach from https://stackoverflow.com/a/17284647/915811, but perhaps there is a way to do this + * effectively without incurring AWS costs (although writing/deleting an empty file to S3 is probably + * miniscule). + * @param s3Bucket + */ + private static boolean verifyS3WritePermissions(AmazonS3 s3Client, String s3Bucket, Request req) { + String key = UUID.randomUUID().toString(); + try { + s3Client.putObject(s3Bucket, key, File.createTempFile("test", ".zip")); + s3Client.deleteObject(s3Bucket, key); + } catch (IOException | AmazonS3Exception e) { + LOG.warn("S3 client cannot write to bucket" + s3Bucket, e); + return false; + } + return true; + } + + /** + * Verify that application can write to S3 bucket. + * + * TODO: Also verify that, with AWS credentials, application can assume instance profile + */ + private static void verifyS3WritePermissions(OtpServer server, Request req) { + // Verify first that this application can write to the S3 bucket, which is needed to write the transit bundle + // file to S3. + if (!verifyS3WritePermissions(FeedStore.s3Client, server.s3Bucket, req)) { + String message = "Application cannot write to specified S3 bucket: " + server.s3Bucket; + logMessageAndHalt(req, 400, message); + } + // TODO: If EC2 info is not null, check that the IAM role ARN is able to write to the S3 bucket. I keep running + // into errors with this code, but will leave it commented out for now. LTR 2019/09/20 +// if (server.ec2Info != null) { +//// InstanceProfile iamInstanceProfile = getIamInstanceProfile(server.ec2Info.iamInstanceProfileArn); +// AWSSecurityTokenServiceClient tokenServiceClient = new +// AWSSecurityTokenServiceClient(FeedStore.getAWSCreds().getCredentials()); +//// AWSSecurityTokenServiceClient tokenServiceClient = new AWSSecurityTokenServiceClient(); +// AssumeRoleRequest request = new AssumeRoleRequest() +// .withRoleArn(server.ec2Info.iamInstanceProfileArn) +// .withDurationSeconds(900) +// .withRoleSessionName("test"); +// AssumeRoleResult result = tokenServiceClient.assumeRole(request); +// Credentials credentials = result.getCredentials(); +// BasicSessionCredentials basicSessionCredentials = new BasicSessionCredentials( +// credentials.getAccessKeyId(), credentials.getSecretAccessKey(), +// credentials.getSessionToken()); +// AmazonS3 temporaryS3Client = AmazonS3ClientBuilder.standard() +// .withCredentials(new AWSStaticCredentialsProvider(basicSessionCredentials)) +//// .withRegion(clientRegion) +// .build(); +// if (!verifyS3WritePermissions(temporaryS3Client, server.s3Bucket, req)) { +// String message = "EC2 IAM role cannot write to specified S3 bucket " + server.s3Bucket; +// logMessageAndHalt(req, 400, message); +// } +// } + } + + /** Validate that AMI exists and value is not empty. */ private static void validateAmiId(String amiId, Request req) { String message = "Server must have valid AMI ID (or field must be empty)"; if (isEmpty(amiId)) return; if (!amiExists(amiId)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); } + /** Determine if AMI ID exists (and is gettable by the application's AWS credentials). */ public static boolean amiExists(String amiId) { try { DescribeImagesRequest request = new DescribeImagesRequest().withImageIds(amiId); @@ -257,11 +318,12 @@ public static boolean amiExists(String amiId) { // Iterate over AMIs to find a matching ID. for (Image image : result.getImages()) if (image.getImageId().equals(amiId)) return true; } catch (AmazonEC2Exception e) { - LOG.info("AMI does not exist.", e); + LOG.warn("AMI does not exist or some error prevented proper checking of the AMI ID.", e); } return false; } + /** Validate that AWS key name (the first part of a .pem key) exists and is not empty. */ private static void validateKeyName(String keyName, Request req) { String message = "Server must have valid key name"; if (isEmpty(keyName)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); @@ -270,15 +332,22 @@ private static void validateKeyName(String keyName, Request req) { logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); } - private static void validateIamRoleArn(String iamRoleArn, Request req) { - String message = "Server must have valid IAM role ARN"; - if (isEmpty(iamRoleArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + /** Get IAM instance profile for the provided role ARN. */ + private static InstanceProfile getIamInstanceProfile (String iamInstanceProfileArn) { ListInstanceProfilesResult result = iam.listInstanceProfiles(); // Iterate over instance profiles. If a matching ARN is found, silently return. - for (InstanceProfile profile: result.getInstanceProfiles()) if (profile.getArn().equals(iamRoleArn)) return; - logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + for (InstanceProfile profile: result.getInstanceProfiles()) if (profile.getArn().equals(iamInstanceProfileArn)) return profile; + return null; + } + + /** Validate IAM instance profile ARN exists and is not empty. */ + private static void validateIamInstanceProfileArn(String iamInstanceProfileArn, Request req) { + String message = "Server must have valid IAM instance profile ARN (e.g., arn:aws:iam::123456789012:instance-profile/otp-ec2-role)."; + if (isEmpty(iamInstanceProfileArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); + if (getIamInstanceProfile(iamInstanceProfileArn) == null) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); } + /** Validate that EC2 security group exists and is not empty. */ private static void validateSecurityGroupId(String securityGroupId, Request req) { String message = "Server must have valid security group ID"; if (isEmpty(securityGroupId)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); @@ -289,6 +358,7 @@ private static void validateSecurityGroupId(String securityGroupId, Request req) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); } + /** Validate that subnet exists and is not empty. */ private static void validateSubnetId(String subnetId, Request req) { String message = "Server must have valid subnet ID"; if (isEmpty(subnetId)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); @@ -300,11 +370,16 @@ private static void validateSubnetId(String subnetId, Request req) { } catch (AmazonEC2Exception e) { logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message, e); } + logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, message); } + /** + * Validate that EC2 instance type (e.g., t2-medium) exists. This value can be empty and will default to + * {@link com.conveyal.datatools.manager.jobs.DeployJob#DEFAULT_INSTANCE_TYPE} at deploy time. + */ private static void validateInstanceType(String instanceType, Request req) { if (instanceType == null) return; - try { + try { InstanceType.fromValue(instanceType); } catch (IllegalArgumentException e) { String message = String.format("Must provide valid instance type (if none provided, defaults to %s).", DEFAULT_INSTANCE_TYPE); @@ -312,6 +387,7 @@ private static void validateInstanceType(String instanceType, Request req) { } } + /** Validate that ELB target group exists and is not empty. */ private static void validateTargetGroup(String targetGroupArn, Request req) { if (isEmpty(targetGroupArn)) logMessageAndHalt(req, HttpStatus.BAD_REQUEST_400, "Invalid value for Target Group ARN."); try { @@ -325,6 +401,9 @@ private static void validateTargetGroup(String targetGroupArn, Request req) { } } + /** + * @return false if string value is empty or null + */ public static boolean isEmpty(String val) { return val == null || "".equals(val); } @@ -335,7 +414,7 @@ public static boolean isEmpty(String val) { public static void register (String apiPrefix) { options(apiPrefix + "secure/servers", (q, s) -> ""); delete(apiPrefix + "secure/servers/:id", ServerController::deleteServer, json::write); - delete(apiPrefix + "secure/servers/:id/ec2", ServerController::terminateEC2Instances, json::write); + delete(apiPrefix + "secure/servers/:id/ec2", ServerController::terminateEC2InstancesForServer, json::write); get(apiPrefix + "secure/servers", ServerController::fetchServers, json::write); post(apiPrefix + "secure/servers", ServerController::createServer, json::write); put(apiPrefix + "secure/servers/:id", ServerController::updateServer, json::write); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/StatusController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/StatusController.java index ae701864f..94873786a 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/StatusController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/StatusController.java @@ -18,6 +18,7 @@ import java.util.stream.Collectors; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; +import static spark.Spark.delete; import static spark.Spark.get; /** @@ -55,6 +56,20 @@ private static MonitorableJob getOneJobRoute(Request req, Response res) { return getJobById(userId, jobId, true); } + /** + * API route that cancels a single job by ID. + */ + // TODO Add ability to cancel job. This requires some changes to how these jobs are executed. It appears that + // only scheduled jobs can be canceled. +// private static MonitorableJob cancelJob(Request req, Response res) { +// String jobId = req.params("jobId"); +// Auth0UserProfile userProfile = req.attribute("user"); +// // FIXME: refactor underscore in user_id methods +// String userId = userProfile.getUser_id(); +// MonitorableJob job = getJobById(userId, jobId, true); +// return job; +// } + /** * Gets a job by user ID and job ID. * @param clearCompleted if true, remove requested job if it has completed or errored @@ -132,5 +147,7 @@ public static void register (String apiPrefix) { // FIXME Change endpoint for all jobs (to avoid overlap with jobId param)? get(apiPrefix + "secure/status/jobs/all", StatusController::getAllJobsRoute, json::write); get(apiPrefix + "secure/status/jobs/:jobId", StatusController::getOneJobRoute, json::write); + // TODO Add ability to cancel job +// delete(apiPrefix + "secure/status/jobs/:jobId", StatusController::cancelJob, json::write); } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index f21125526..768d96dce 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -14,12 +14,11 @@ import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.Tag; -import com.amazonaws.services.ec2.model.TerminateInstancesRequest; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; -import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @@ -30,6 +29,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.Serializable; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; @@ -38,6 +38,7 @@ import java.nio.channels.WritableByteChannel; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -55,10 +56,13 @@ import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.controllers.api.ServerController; import com.conveyal.datatools.manager.models.Deployment; +import com.conveyal.datatools.manager.models.EC2Info; import com.conveyal.datatools.manager.models.EC2InstanceSummary; import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.StringUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.codec.binary.Base64; import org.slf4j.Logger; @@ -79,7 +83,10 @@ public class DeployJob extends MonitorableJob { private static final String bundlePrefix = "bundles"; public static final String DEFAULT_INSTANCE_TYPE = "t2.medium"; private static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; - public static final String DEFAULT_AMI_ID = DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); + private static final String DEFAULT_AMI_ID = DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); + private static final String OTP_GRAPH_FILENAME = "Graph.obj"; + public static final String BUNDLE_DOWNLOAD_COMPLETE_FILE = "BUNDLE_DOWNLOAD_COMPLETE"; + private static final long TEN_MINUTES_IN_MILLISECONDS = 10 * 60 * 1000; /** * S3 bucket to upload deployment to. If not null, uses {@link OtpServer#s3Bucket}. Otherwise, defaults to * {@link DataManager#feedBucket} @@ -118,6 +125,14 @@ public String getServerId () { return otpServer.id; } + public Deployment getDeployment() { + return deployment; + } + + public OtpServer getOtpServer() { + return otpServer; + } + public DeployJob(Deployment deployment, String owner, OtpServer otpServer) { // TODO add new job type or get rid of enum in favor of just using class names super(owner, "Deploying " + deployment.name, JobType.DEPLOY_TO_OTP); @@ -216,12 +231,13 @@ public void jobLogic () { * Upload to S3 the transit data bundle zip that contains GTFS zip files, OSM data, and config files. */ private void uploadBundleToS3() throws InterruptedException, AmazonClientException { - status.message = "Uploading to s3://" + s3Bucket; + AmazonS3URI uri = new AmazonS3URI(getS3BundleURI()); + String bucket = uri.getBucket(); + status.message = "Uploading bundle to " + getS3BundleURI(); status.uploadingS3 = true; - String key = getS3BundleKey(); - LOG.info("Uploading deployment {} to s3://{}/{}", deployment.name, s3Bucket, key); + LOG.info("Uploading deployment {} to {}", deployment.name, uri.toString()); TransferManager tx = TransferManagerBuilder.standard().withS3Client(FeedStore.s3Client).build(); - final Upload upload = tx.upload(s3Bucket, key, deploymentTempFile); + final Upload upload = tx.upload(bucket, uri.getKey(), deploymentTempFile); upload.addProgressListener( (ProgressListener) progressEvent -> status.percentUploaded = upload.getProgress().getPercentTransferred() @@ -236,18 +252,19 @@ private void uploadBundleToS3() throws InterruptedException, AmazonClientExcepti // copy to [name]-latest.zip String copyKey = getLatestS3BundleKey(); - CopyObjectRequest copyObjRequest = new CopyObjectRequest(s3Bucket, key, s3Bucket, copyKey); + CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucket, uri.getKey(), uri.getBucket(), copyKey); FeedStore.s3Client.copyObject(copyObjRequest); - LOG.info("Copied to s3://{}/{}", s3Bucket, copyKey); - LOG.info("Uploaded to s3://{}/{}", s3Bucket, getS3BundleKey()); + LOG.info("Copied to s3://{}/{}", bucket, copyKey); + LOG.info("Uploaded to {}", getS3BundleURI()); status.update("Upload to S3 complete.", status.percentComplete + 10); status.uploadingS3 = false; } + /** + * Builds the OTP graph over wire, i.e., send the data over an HTTP POST request to boot/replace the existing graph + * using the OTP Routers#buildGraphOverWire endpoint. + */ private boolean buildGraphOverWire() { - // figure out what router we're using - String router = deployment.routerId != null ? deployment.routerId : "default"; - // Send the deployment file over the wire to each OTP server. for (String rawUrl : otpServer.internalUrl) { status.message = "Deploying to " + rawUrl; @@ -256,7 +273,7 @@ private boolean buildGraphOverWire() { URL url; try { - url = new URL(rawUrl + "/routers/" + router); + url = new URL(rawUrl + "/routers/" + getRouterId()); } catch (MalformedURLException e) { statusMessage = String.format("Malformed deployment URL %s", rawUrl); LOG.error(statusMessage); @@ -384,12 +401,13 @@ private boolean buildGraphOverWire() { return true; } - private String getS3BundleKey() { - return String.format("%s/%s/%s.zip", bundlePrefix, deployment.projectId, this.jobId); + private String getS3BundleURI() { + return joinToS3FolderURI("bundle.zip"); } - private String getLatestS3BundleKey() { - return String.format("%s/%s/%s-latest.zip", bundlePrefix, deployment.projectId, deployment.parentProject().name.toLowerCase()); + private String getLatestS3BundleKey() { + String name = StringUtils.getCleanName(deployment.parentProject().name.toLowerCase()); + return String.format("%s/%s/%s-latest.zip", bundlePrefix, deployment.projectId, name); } @Override @@ -404,20 +422,25 @@ public void jobFinished () { if (!status.error) { // Update status with successful completion state only if no error was encountered. status.update(false, "Deployment complete!", 100, true); - // Store the target server in the deployedTo field. - LOG.info("Updating deployment target to {} id={}", otpServer.id, deployment.id); - Persistence.deployments.updateField(deployment.id, "deployedTo", otpServer.id); - // Update last deployed field. - Persistence.deployments.updateField(deployment.id, "lastDeployed", new Date()); - message = String.format("Deployment %s successfully deployed to %s", deployment.name, otpServer.publicUrl); + // Store the target server in the deployedTo field and set last deployed time. + LOG.info("Updating deployment target and deploy time."); + deployment.deployedTo = otpServer.id; + deployment.deployJobSummaries.add(0, new DeploySummary(this)); + Persistence.deployments.replace(deployment.id, deployment); + long durationMinutes = TimeUnit.MILLISECONDS.toMinutes(status.duration); + message = String.format("Deployment %s successfully deployed to %s in %s minutes.", deployment.name, otpServer.publicUrl, durationMinutes); } else { - message = String.format("WARNING: Deployment %s failed to deploy to %s", deployment.name, otpServer.publicUrl); + message = String.format("WARNING: Deployment %s failed to deploy to %s. Error: %s", deployment.name, otpServer.publicUrl, status.message); } // Send notification to those subscribed to updates for the deployment. NotifyUsersForSubscriptionJob.createNotification("deployment-updated", deployment.id, message); } - + /** + * Start up EC2 instances as trip planning servers running on the provided ELB. After monitoring the server statuses + * and verifying that they are running, remove the previous EC2 instances that were assigned to the ELB and terminate + * them. + */ private void replaceEC2Servers() { try { // Track any previous instances running for the server we're deploying to in order to de-register and @@ -425,23 +448,52 @@ private void replaceEC2Servers() { List previousInstances = otpServer.retrieveEC2InstanceSummaries(); // First start graph-building instance and wait for graph to successfully build. status.message = "Starting up graph building EC2 instance"; - List instances = startEC2Instances(1); + List instances = startEC2Instances(1, false); + // Exit if an error was encountered. + if (status.error || instances.size() == 0) { + ServerController.terminateInstances(getIds(instances)); + return; + } status.message = "Waiting for graph build to complete..."; - MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob(owner, deployment, instances.get(0), otpServer); + MonitorServerStatusJob monitorInitialServerJob = new MonitorServerStatusJob(owner, this, instances.get(0), false); monitorInitialServerJob.run(); + status.update("Graph build is complete!", 50); + // If only building graph, job is finished. Note: the graph building EC2 instance should automatically shut + // itself down if this flag is turned on (happens in user data). We do not want to proceed with the rest of + // the job which would shut down existing servers running for the deployment. + if (deployment.buildGraphOnly) { + status.update("Graph build is complete!", 100); + return; + } + Persistence.deployments.replace(deployment.id, deployment); + if (monitorInitialServerJob.status.error) { + // If an error occurred while monitoring the initial server, fail this job and instruct user to inspect + // build logs. + statusMessage = "Error encountered while building graph. Inspect build logs."; + LOG.error(statusMessage); + status.fail(statusMessage); + ServerController.terminateInstances(getIds(instances)); + return; + } // Spin up remaining servers which will download the graph from S3. int remainingServerCount = otpServer.ec2Info.instanceCount <= 0 ? 0 : otpServer.ec2Info.instanceCount - 1; + List remainingServerMonitorJobs = new ArrayList<>(); + List remainingInstances = new ArrayList<>(); if (remainingServerCount > 0) { // Spin up remaining EC2 instances. status.message = String.format("Spinning up remaining %d instance(s).", remainingServerCount); - List remainingInstances = startEC2Instances(remainingServerCount); - instances.addAll(remainingInstances); + remainingInstances.addAll(startEC2Instances(remainingServerCount, true)); + if (remainingInstances.size() == 0 || status.error) { + ServerController.terminateInstances(getIds(remainingInstances)); + return; + } // Create new thread pool to monitor server setup so that the servers are monitored in parallel. ExecutorService service = Executors.newFixedThreadPool(remainingServerCount); for (Instance instance : remainingInstances) { // Note: new instances are added - MonitorServerStatusJob monitorServerStatusJob = new MonitorServerStatusJob(owner, deployment, instance, otpServer); + MonitorServerStatusJob monitorServerStatusJob = new MonitorServerStatusJob(owner, this, instance, true); + remainingServerMonitorJobs.add(monitorServerStatusJob); service.submit(monitorServerStatusJob); } // Shutdown thread pool once the jobs are completed and wait for its termination. Once terminated, we can @@ -449,29 +501,28 @@ private void replaceEC2Servers() { service.shutdown(); service.awaitTermination(4, TimeUnit.HOURS); } + // Check if any of the monitor jobs encountered any errors and terminate the job's associated instance. + for (MonitorServerStatusJob job : remainingServerMonitorJobs) { + if (job.status.error) { + String id = job.getInstanceId(); + LOG.warn("Error encountered while monitoring server {}. Terminating.", id); + remainingInstances.removeIf(instance -> instance.getInstanceId().equals(id)); + ServerController.terminateInstances(id); + } + } + // Add all servers that did not encounter issues to list for registration with ELB. + instances.addAll(remainingInstances); String finalMessage = "Server setup is complete!"; // Get EC2 servers running that are associated with this server. - List instanceIds = previousInstances.stream() + List previousInstanceIds = previousInstances.stream() .filter(instance -> "running".equals(instance.state.getName())) .map(instance -> instance.instanceId) .collect(Collectors.toList()); - if (instanceIds.size() > 0) { - // Deregister old instances from load balancer. (Note: new instances are registered with load balancer in - // MonitorServerStatusJob.) - LOG.info("De-registering instances from load balancer {}", instanceIds); - TargetDescription[] targetDescriptions = instanceIds.stream() - .map(id -> new TargetDescription().withId(id)) - .toArray(TargetDescription[]::new); - DeregisterTargetsRequest deregisterTargetsRequest = new DeregisterTargetsRequest() - .withTargetGroupArn(otpServer.ec2Info.targetGroupArn) - .withTargets(targetDescriptions); - AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); - elb.deregisterTargets(deregisterTargetsRequest); - try { - ServerController.terminateInstances(instanceIds); - } catch (AmazonEC2Exception e) { - LOG.warn("Could not terminate EC2 instances {}", instanceIds); - finalMessage = String.format("Server setup is complete! (WARNING: Could not terminate previous EC2 instances: %s", instanceIds); + if (previousInstanceIds.size() > 0) { + boolean success = deRegisterAndTerminateInstances(previousInstanceIds); + // If there was a problem during de-registration/termination, notify via status message. + if (!success) { + finalMessage = String.format("Server setup is complete! (WARNING: Could not terminate previous EC2 instances: %s", previousInstanceIds); } } // Job is complete. @@ -482,19 +533,50 @@ private void replaceEC2Servers() { } } + /** + * De-register instances from the load balancer and terminate the instanced. + * + * (Note: new instances are registered with load balancer in {@link MonitorServerStatusJob}.) + */ + private boolean deRegisterAndTerminateInstances(List instanceIds) { + LOG.info("De-registering instances from load balancer {}", instanceIds); + TargetDescription[] targetDescriptions = instanceIds.stream() + .map(id -> new TargetDescription().withId(id)) + .toArray(TargetDescription[]::new); + DeregisterTargetsRequest deregisterTargetsRequest = new DeregisterTargetsRequest() + .withTargetGroupArn(otpServer.ec2Info.targetGroupArn) + .withTargets(targetDescriptions); + AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); + elb.deregisterTargets(deregisterTargetsRequest); + try { + ServerController.terminateInstances(instanceIds); + } catch (AmazonEC2Exception e) { + LOG.warn("Could not terminate EC2 instances {}", instanceIds); + return false; + } + return true; + } + /** * Start the specified number of EC2 instances based on the {@link OtpServer#ec2Info}. * @param count number of EC2 instances to start * @return a list of the instances is returned once the public IP addresses have been assigned + * + * TODO: Booting up R5 servers has not been fully tested. */ - private List startEC2Instances(int count) { + private List startEC2Instances(int count, boolean graphAlreadyBuilt) { String instanceType = otpServer.ec2Info.instanceType == null ? DEFAULT_INSTANCE_TYPE : otpServer.ec2Info.instanceType; // User data should contain info about: // 1. Downloading GTFS/OSM info (s3) // 2. Time to live until shutdown/termination (for test servers) // 3. Hosting / nginx - // FIXME: Allow for r5 servers to be created. - String userData = constructUserData(); + String userData = constructUserData(graphAlreadyBuilt); + // Failure was encountered while constructing user data. + if (userData == null) { + // Fail job if it is not already failed. + if (!status.error) status.fail("Error constructing EC2 user data."); + return Collections.EMPTY_LIST; + } // The subnet ID should only change if starting up a server in some other AWS account. This is not // likely to be a requirement. // Define network interface so that a public IP can be associated with server. @@ -522,7 +604,7 @@ private List startEC2Instances(int count) { .withInstanceType(instanceType) .withMinCount(count) .withMaxCount(count) - .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(otpServer.ec2Info.iamRoleArn)) + .withIamInstanceProfile(new IamInstanceProfileSpecification().withArn(otpServer.ec2Info.iamInstanceProfileArn)) .withImageId(amiId) .withKeyName(otpServer.ec2Info.keyName) // This will have the instance terminate when it is shut down. @@ -539,9 +621,9 @@ private List startEC2Instances(int count) { waiter.run(new WaiterParameters<>(new DescribeInstanceStatusRequest().withInstanceIds(instanceIds))); LOG.info("Instance status is OK after {} ms", (System.currentTimeMillis() - beginWaiting)); } catch (Exception e) { - LOG.error("Waiter for instance status check failed.", e); - status.fail("Waiter for instance status check failed."); - // FIXME: Terminate instance??? + statusMessage = "Waiter for instance status check failed. You may need to terminate the failed instances."; + LOG.error(statusMessage, e); + status.fail(statusMessage); return Collections.EMPTY_LIST; } for (Instance instance : instances) { @@ -555,6 +637,7 @@ private List startEC2Instances(int count) { .withTags(new Tag("deploymentId", deployment.id)) .withTags(new Tag("jobId", this.jobId)) .withTags(new Tag("serverId", otpServer.id)) + .withTags(new Tag("routerId", getRouterId())) .withResources(instance.getInstanceId()) ); } @@ -575,98 +658,146 @@ private List startEC2Instances(int count) { } try { int sleepTimeMillis = 10000; - LOG.info("Waiting {} seconds...", sleepTimeMillis / 1000); + LOG.info("Waiting {} seconds to perform another public IP address check...", sleepTimeMillis / 1000); Thread.sleep(sleepTimeMillis); } catch (InterruptedException e) { e.printStackTrace(); } + if (System.currentTimeMillis() - status.startTime > TEN_MINUTES_IN_MILLISECONDS) { + status.fail("Job timed out due to public IP assignment taking longer than ten minutes!"); + return updatedInstances; + } } LOG.info("Public IP addresses have all been assigned. {}", instanceIpAddresses.values().toString()); return updatedInstances; } + /** + * @return the router ID for this deployment (defaults to "default") + */ + private String getRouterId() { + return deployment.routerId == null ? "default" : deployment.routerId; + } + /** * Construct the user data script (as string) that should be provided to the AMI and executed upon EC2 instance * startup. */ - private String constructUserData() { + private String constructUserData(boolean graphAlreadyBuilt) { // Prefix/name of JAR file (WITHOUT .jar) String jarName = deployment.r5 ? deployment.r5Version : deployment.otpVersion; if (jarName == null) { + if (deployment.r5) deployment.r5Version = DEFAULT_R5_VERSION; + else deployment.otpVersion = DEFAULT_OTP_VERSION; // If there is no version specified, use the default (and persist value). - jarName = deployment.r5 ? DEFAULT_R5_VERSION : DEFAULT_OTP_VERSION; - Persistence.deployments.updateField(deployment.id, deployment.r5 ? "r5Version" : "otpVersion", jarName); + jarName = deployment.r5 ? deployment.r5Version : deployment.otpVersion; + Persistence.deployments.replace(deployment.id, deployment); } - String tripPlanner = deployment.r5 ? "r5" : "otp"; String s3JarBucket = deployment.r5 ? "r5-builds" : "opentripplanner-builds"; - String s3JarUrl = String.format("https://%s.s3.amazonaws.com/%s.jar", s3JarBucket, jarName); - // TODO Check that jar URL exists? - String jarDir = String.format("/opt/%s", tripPlanner); - String s3BundlePath = String.format("s3://%s/%s", s3Bucket, getS3BundleKey()); - // Note, an AmazonS3Exception will be thrown by S3 if the object does not exist. This is a feature to avoid - // revealing to non-authorized users whether the object actually exists. So, as long as permissions are - // configured correctly a 403 indicates it does not exist. - boolean graphAlreadyBuilt = false; - try{ - graphAlreadyBuilt = FeedStore.s3Client.doesObjectExist(s3Bucket, getS3GraphKey()); - } catch (AmazonS3Exception e) { - LOG.warn("Error checking if graph object exists. This is likely because it does not exist.", e); + String s3JarKey = jarName + ".jar"; + // If jar does not exist in bucket, fail job. + if (!FeedStore.s3Client.doesObjectExist(s3JarBucket, s3JarKey)) { + status.fail(String.format("Requested jar does not exist at s3://%s/%s", s3JarBucket, s3JarKey)); + return null; } + String s3JarUrl = String.format("https://%s.s3.amazonaws.com/%s", s3JarBucket, s3JarKey); + String jarDir = String.format("/opt/%s", getTripPlannerString()); List lines = new ArrayList<>(); String routerName = "default"; - String routerDir = String.format("/var/%s/graphs/%s", tripPlanner, routerName); - // BEGIN USER DATA + String routerDir = String.format("/var/%s/graphs/%s", getTripPlannerString(), routerName); + //////////////// BEGIN USER DATA lines.add("#!/bin/bash"); // Send trip planner logs to LOGFILE - lines.add(String.format("BUILDLOGFILE=/var/log/%s-build.log", tripPlanner)); - lines.add(String.format("LOGFILE=/var/log/%s.log", tripPlanner)); + lines.add(String.format("BUILDLOGFILE=/var/log/%s", getBuildLogFilename())); + lines.add(String.format("LOGFILE=/var/log/%s.log", getTripPlannerString())); + lines.add("USERDATALOG=/var/log/user-data.log"); // Log user data setup to /var/log/user-data.log - lines.add("exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1"); + lines.add("exec > >(tee $USERDATALOG|logger -t user-data -s 2>/dev/console) 2>&1"); // Create the directory for the graph inputs. lines.add(String.format("mkdir -p %s", routerDir)); lines.add(String.format("chown ubuntu %s", routerDir)); - // Remove the current inputs and replace with inputs from S3. + // Remove the current inputs from router directory. lines.add(String.format("rm -rf %s/*", routerDir)); - lines.add(String.format("aws s3 --region us-east-1 cp %s /tmp/bundle.zip", s3BundlePath)); - lines.add(String.format("unzip /tmp/bundle.zip -d %s", routerDir)); - // FIXME: Add ability to fetch custom bikeshare.xml file (CarFreeAtoZ) - if (false) { - lines.add(String.format("wget -O %s/bikeshare.xml ${config.bikeshareFeed}", routerDir)); - lines.add(String.format("printf \"{\\n bikeRentalFile: \"bikeshare.xml\"\\n}\" >> %s/build-config.json\"", routerDir)); - } // Download trip planner JAR. lines.add(String.format("mkdir -p %s", jarDir)); lines.add(String.format("wget %s -O %s/%s.jar", s3JarUrl, jarDir, jarName)); if (graphAlreadyBuilt) { lines.add("echo 'downloading graph from s3'"); - // Download Graph from S3 and spin up trip planner. - lines.add(String.format("aws s3 --region us-east-1 cp %s %s/Graph.obj", getS3GraphPath(), routerDir)); + // Download Graph from S3. + lines.add(String.format("aws s3 --region us-east-1 cp %s %s/%s ", getS3GraphURI(), routerDir, OTP_GRAPH_FILENAME)); } else { + // Download data bundle from S3. + lines.add(String.format("aws s3 --region us-east-1 cp %s /tmp/bundle.zip", getS3BundleURI())); + // Determine if bundle download was successful. + lines.add("[ -f /tmp/bundle.zip ] && BUNDLE_STATUS='SUCCESS' || BUNDLE_STATUS='FAILURE'"); + // Create and upload file with bundle status to notify Data Tools that download is complete. + lines.add(String.format("echo $BUNDLE_STATUS > /tmp/%s", BUNDLE_DOWNLOAD_COMPLETE_FILE)); + lines.add(String.format("aws s3 --region us-east-1 cp /tmp/%s %s", BUNDLE_DOWNLOAD_COMPLETE_FILE, joinToS3FolderURI(BUNDLE_DOWNLOAD_COMPLETE_FILE))); + // Put unzipped bundle data into router directory. + lines.add(String.format("unzip /tmp/bundle.zip -d %s", routerDir)); + // FIXME: Add ability to fetch custom bikeshare.xml file (CarFreeAtoZ) + if (false) { + lines.add(String.format("wget -O %s/bikeshare.xml ${config.bikeshareFeed}", routerDir)); + lines.add(String.format("printf \"{\\n bikeRentalFile: \"bikeshare.xml\"\\n}\" >> %s/build-config.json\"", routerDir)); + } lines.add("echo 'starting graph build'"); // Build the graph if Graph object (presumably this is the first instance to be started up). if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu java -Xmx6G -jar %s/%s.jar point --build %s", jarDir, jarName, routerDir)); else lines.add(String.format("sudo -H -u ubuntu java -jar %s/%s.jar --build %s > $BUILDLOGFILE 2>&1", jarDir, jarName, routerDir)); - // Upload the graph to S3. - if (!deployment.r5) lines.add(String.format("aws s3 --region us-east-1 cp %s/Graph.obj %s", routerDir, getS3GraphPath())); + // Upload the graph and build log file to S3. + if (!deployment.r5) { + lines.add(String.format("aws s3 --region us-east-1 cp %s/%s %s ", routerDir, OTP_GRAPH_FILENAME, getS3GraphURI())); + String s3BuildLogPath = joinToS3FolderURI(getBuildLogFilename()); + lines.add(String.format("aws s3 --region us-east-1 cp $BUILDLOGFILE %s ", s3BuildLogPath)); + } } + // Upload user data log. + lines.add("instance_id=`curl http://169.254.169.254/latest/meta-data/instance-id`"); + lines.add(String.format("aws s3 --region us-east-1 cp $USERDATALOG %s/${instance_id}.log", getS3FolderURI().toString())); if (deployment.buildGraphOnly) { + // If building graph only, tell the instance to shut itself down after the graph build (and log upload) is + // complete. lines.add("echo 'shutting down server (build graph only specified in deployment target)'"); lines.add("sudo poweroff"); } else { + // Otherwise, kick off the application. lines.add("echo 'kicking off trip planner (logs at $LOGFILE)'"); - // Kick off the application. if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu nohup java -Xmx6G -Djava.util.Arrays.useLegacyMergeSort=true -jar %s/%s.jar point --isochrones %s > /var/log/r5.out 2>&1&", jarDir, jarName, routerDir)); else lines.add(String.format("sudo -H -u ubuntu nohup java -jar %s/%s.jar --server --bindAddress 127.0.0.1 --router default > $LOGFILE 2>&1 &", jarDir, jarName)); } + // Return the entire user data script as a single string. return String.join("\n", lines); } - private String getS3GraphKey() { - return String.format("%s/%s/Graph.obj", deployment.projectId, this.jobId); + private String getBuildLogFilename() { + return String.format("%s-build.log", getTripPlannerString()); } - private String getS3GraphPath() { - return String.format("s3://%s/%s", otpServer.s3Bucket, getS3GraphKey()); + private String getTripPlannerString() { + return deployment.r5 ? "r5" : "otp"; + } + + @JsonIgnore + public String getJobRelativePath() { + return String.join("/", bundlePrefix, deployment.projectId, deployment.id, this.jobId); + } + + @JsonIgnore + public AmazonS3URI getS3FolderURI() { + return new AmazonS3URI(String.format("s3://%s/%s", otpServer.s3Bucket, getJobRelativePath())); + } + + @JsonIgnore + public String getS3GraphURI() { + return joinToS3FolderURI(OTP_GRAPH_FILENAME); + } + + /** Join list of paths to S3 URI for job folder to create a fully qualified URI (e.g., s3://bucket/path/to/file). */ + private String joinToS3FolderURI(CharSequence... paths) { + List pathList = new ArrayList<>(); + pathList.add(getS3FolderURI().toString()); + pathList.addAll(Arrays.asList(paths)); + return String.join("/", pathList); } /** @@ -693,4 +824,34 @@ public static class DeployStatus extends Status { public String baseUrl; } + + /** + * Contains details about a specific deployment job in order to preserve and recall this info after the job has + * completed. + */ + public static class DeploySummary implements Serializable { + private static final long serialVersionUID = 1L; + public long duration; + public String s3Bucket; + public String jobId; + /** URL for build log file from latest deploy job. */ + public String buildArtifactsFolder; + public String otpVersion; + public EC2Info ec2Info; + public long startTime; + public long finishTime = System.currentTimeMillis(); + + /** Empty constructor for serialization */ + public DeploySummary () { } + + public DeploySummary (DeployJob job) { + this.ec2Info = job.otpServer.ec2Info; + this.otpVersion = job.deployment.otpVersion; + this.jobId = job.jobId; + this.s3Bucket = job.s3Bucket; + this.startTime = job.status.startTime; + this.duration = job.status.duration; + this.buildArtifactsFolder = job.getS3FolderURI().toString(); + } + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 2aa392665..3f7b8c05c 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -10,9 +10,13 @@ import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; +import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.S3Object; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.OtpServer; +import com.conveyal.datatools.manager.persistence.FeedStore; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -30,62 +34,110 @@ */ public class MonitorServerStatusJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(MonitorServerStatusJob.class); + private final DeployJob deployJob; private final Deployment deployment; private final Instance instance; + private final boolean graphAlreadyBuilt; private final OtpServer otpServer; private final AmazonEC2 ec2 = AmazonEC2Client.builder().build(); private final CloseableHttpClient httpClient = HttpClients.createDefault(); // If the job takes longer than XX seconds, fail the job. private static final int TIMEOUT_MILLIS = 60 * 60 * 1000; // One hour + private static final int DELAY_SECONDS = 5; + private final long startTime; + public long graphBuildSeconds; - public MonitorServerStatusJob(String owner, Deployment deployment, Instance instance, OtpServer otpServer) { + public MonitorServerStatusJob(String owner, DeployJob deployJob, Instance instance, boolean graphAlreadyBuilt) { super( owner, String.format("Monitor server setup %s", instance.getPublicIpAddress()), JobType.MONITOR_SERVER_STATUS ); - this.deployment = deployment; + this.deployJob = deployJob; + this.deployment = deployJob.getDeployment(); + this.otpServer = deployJob.getOtpServer(); this.instance = instance; - this.otpServer = otpServer; + this.graphAlreadyBuilt = graphAlreadyBuilt; status.message = "Checking server status..."; + startTime = System.currentTimeMillis(); + } + + @JsonProperty + public String getInstanceId () { + return instance != null ? instance.getInstanceId() : null; + } + + @JsonProperty + public String getDeploymentId () { + return deployJob.getDeploymentId(); } @Override public void jobLogic() { - long startTime = System.currentTimeMillis(); // Get OTP URL for instance to check for availability. - String otpUrl = String.format("http://%s/otp", instance.getPublicIpAddress()); - boolean otpIsRunning = false, routerIsAvailable = false; - // Progressively check status of OTP server - if (deployment.buildGraphOnly) { - // FIXME No need to check that OTP is running. Just check to see that the graph is built. -// FeedStore.s3Client.doesObjectExist(otpServer.s3Bucket, ); - } - // First, check that OTP has started up. - status.update("Instance status is OK. Waiting for OTP...", 20); - while (!otpIsRunning) { - LOG.info("Checking that OTP is running on server at {}.", otpUrl); - // If the request is successful, the OTP instance has started. - otpIsRunning = checkForSuccessfulRequest(otpUrl, 5); - if (System.currentTimeMillis() - startTime > TIMEOUT_MILLIS) { - status.fail(String.format("Job timed out while monitoring setup for server %s", instance.getInstanceId())); + boolean routerIsAvailable = false; + // If graph was not already built by a previous server, wait for it to build. + if (!graphAlreadyBuilt) { + boolean bundleIsDownloaded = false, graphBuildIsComplete = false; + // Progressively check status of OTP server + if (deployment.buildGraphOnly) { + // No need to check that OTP is running. Just check to see that the graph is built. + bundleIsDownloaded = true; + routerIsAvailable = true; + } + // First, check that OTP has started up. + status.update("Prepping for graph build...", 20); + while (!bundleIsDownloaded) { + // If the request is successful, the OTP instance has started. + wait("bundle download check"); + bundleIsDownloaded = isBundleDownloaded(); + if (jobHasTimedOut()) { + status.fail(String.format("Job timed out while checking for server bundle download status (%s)", instance.getInstanceId())); + return; + } + } + // Check status of bundle download and fail job if there was a failure. + String bundleStatus = FeedStore.s3Client.getObjectAsString(otpServer.s3Bucket, getBundleStatusKey()); + if (bundleStatus == null || !bundleStatus.contains("SUCCESS")) { + status.fail("Failure encountered while downloading transit bundle."); + return; + } + status.update("Building graph...", 30); + long graphBuildStartTime = System.currentTimeMillis(); + while (!graphBuildIsComplete) { + // If the request is successful, the OTP instance has started. + wait("graph build check"); + graphBuildIsComplete = isGraphBuilt(); + if (jobHasTimedOut()) { + status.fail(String.format("Job timed out while waiting for graph build (%s)", instance.getInstanceId())); + return; + } + } + graphBuildSeconds = (System.currentTimeMillis() - graphBuildStartTime) / 1000; + String message = String.format("Graph build completed in %d seconds!", graphBuildSeconds); + LOG.info(message); + if (deployment.buildGraphOnly) { + status.update(false, message, 100); return; } } - String otpMessage = String.format("OTP is running on server %s (%s). Building graph...", instance.getInstanceId(), otpUrl); - LOG.info(otpMessage); - status.update(otpMessage, 30); + status.update("Loading graph...", 40); // Once this is confirmed, check for the existence of the router, which will indicate that the graph build is // complete. - String routerUrl = String.format("%s/routers/default", otpUrl); + String routerUrl = String.format("http://%s/otp/routers/default", instance.getPublicIpAddress()); while (!routerIsAvailable) { LOG.info("Checking that router is available (i.e., graph build or read is finished) at {}", routerUrl); // If the request was successful, the graph build is complete! // TODO: Substitute in specific router ID? Or just default to... default. - routerIsAvailable = checkForSuccessfulRequest(routerUrl, 5); + wait("trip planner to start up"); + routerIsAvailable = checkForSuccessfulRequest(routerUrl); + if (jobHasTimedOut()) { + status.fail(String.format("Job timed out while waiting for trip planner to start up (%s)", instance.getInstanceId())); + return; + } } - status.update(String.format("Graph build completed on server %s (%s).", instance.getInstanceId(), routerUrl), 90); - if (otpServer.ec2Info.targetGroupArn != null) { + status.update("Graph loaded!", 90); + if (otpServer.ec2Info != null && otpServer.ec2Info.targetGroupArn != null) { // After the router is available, the EC2 instance can be registered with the load balancer. // REGISTER INSTANCE WITH LOAD BALANCER AmazonElasticLoadBalancing elbClient = AmazonElasticLoadBalancingClient.builder().build(); @@ -94,24 +146,56 @@ public void jobLogic() { .withTargets(new TargetDescription().withId(instance.getInstanceId())); elbClient.registerTargets(registerTargetsRequest); // FIXME how do we know it was successful? - String message = String.format("Server %s successfully registered with load balancer %s", instance.getInstanceId(), otpServer.ec2Info.targetGroupArn); + String message = String.format("Server successfully registered with load balancer %s. OTP running at %s", otpServer.ec2Info.targetGroupArn, routerUrl); LOG.info(message); status.update(false, message, 100, true); } else { - LOG.info("There is no load balancer under which to register ec2 instance {}.", instance.getInstanceId()); + String message = String.format("There is no load balancer under which to register ec2 instance %s.", instance.getInstanceId()); + LOG.error(message); + status.fail(message); } } + /** Determine if job has passed time limit for its run time. */ + private boolean jobHasTimedOut() { + long runTime = System.currentTimeMillis() - startTime; + return runTime > TIMEOUT_MILLIS; + } + /** - * Checks the provided URL for a successful response (i.e., HTTP status code is 200). + * Checks for Graph object on S3. */ - private boolean checkForSuccessfulRequest(String url, int delaySeconds) { - // Wait for the specified seconds before the request is initiated. + private boolean isGraphBuilt() { + AmazonS3URI uri = new AmazonS3URI(deployJob.getS3GraphURI()); + LOG.info("Checking for graph at {}", uri.toString()); + return FeedStore.s3Client.doesObjectExist(uri.getBucket(), uri.getKey()); + } + + /** Have the current thread sleep for a few seconds in order to pause during a while loop. */ + private void wait(String waitingFor) { try { - Thread.sleep(1000 * delaySeconds); + LOG.info("Waiting {} seconds for {}", DELAY_SECONDS, waitingFor); + Thread.sleep(1000 * DELAY_SECONDS); } catch (InterruptedException e) { e.printStackTrace(); } + } + + private String getBundleStatusKey () { + return String.join("/", deployJob.getJobRelativePath(), DeployJob.BUNDLE_DOWNLOAD_COMPLETE_FILE); + } + + /** Check if the bundle download completed file has been uploaded to S3. */ + private boolean isBundleDownloaded() { + String key = getBundleStatusKey(); + LOG.info("Checking for bundle complete at s3://{}/{}", otpServer.s3Bucket, key); + return FeedStore.s3Client.doesObjectExist(otpServer.s3Bucket, key); + } + + /** + * Checks the provided URL for a successful response (i.e., HTTP status code is 200). + */ + private boolean checkForSuccessfulRequest(String url) { HttpGet httpGet = new HttpGet(url); try (CloseableHttpResponse response = httpClient.execute(httpGet)) { HttpEntity entity = response.getEntity(); diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index a927d7cbd..91a14b9d5 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -3,6 +3,7 @@ import com.amazonaws.services.ec2.model.Filter; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.controllers.api.DeploymentController; +import com.conveyal.datatools.manager.jobs.DeployJob; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.StringUtils; import com.conveyal.datatools.manager.utils.json.JsonManager; @@ -64,6 +65,8 @@ public class Deployment extends Model implements Serializable { /** What server is this currently deployed to? */ public String deployedTo; + public List deployJobSummaries = new ArrayList<>(); + @JsonView(JsonViews.DataDump.class) public String projectId; @@ -147,7 +150,15 @@ public void storeFeedVersions(Collection versions) { public boolean r5; /** Date when the deployment was last deployed to a server */ - public Date lastDeployed; + @JsonProperty("lastDeployed") + public Date retrieveLastDeployed () { + return latest() != null ? new Date(latest().finishTime) : null; + } + + /** Get latest deployment summary. */ + public DeployJob.DeploySummary latest () { + return deployJobSummaries.size() > 0 ? deployJobSummaries.get(0) : null; + } /** * The routerId of this deployment diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java index fc5d406c8..fcbd37aa7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2Info.java @@ -27,8 +27,8 @@ public EC2Info () {} public String securityGroupId; /** The Amazon machine image (AMI) to be used for the OTP EC2 machines. */ public String amiId; - /** The IAM role ARN that the OTP EC2 server should assume. */ - public String iamRoleArn; + /** The IAM instance profile ARN that the OTP EC2 server should assume. For example, arn:aws:iam::123456789012:instance-profile/otp-ec2-role */ + public String iamInstanceProfileArn; /** The AWS key file (.pem) that should be used to set up OTP EC2 servers (gives a way for admins to SSH into machine). */ public String keyName; /** The target group to deploy new EC2 instances to. */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java b/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java index d27ad9005..ff53ac5f1 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java +++ b/src/main/java/com/conveyal/datatools/manager/models/EC2InstanceSummary.java @@ -20,6 +20,7 @@ public class EC2InstanceSummary implements Serializable { public String instanceId; public String imageId; public String projectId; + public String jobId; public String deploymentId; public String name; public InstanceState state; @@ -39,17 +40,12 @@ public EC2InstanceSummary (Instance ec2Instance) { imageId = ec2Instance.getImageId(); List tags = ec2Instance.getTags(); // Set project and deployment ID if they exist. - String projectId = null; - String deploymentId = null; - String name = null; for (Tag tag : tags) { if (tag.getKey().equals("projectId")) projectId = tag.getValue(); if (tag.getKey().equals("deploymentId")) deploymentId = tag.getValue(); + if (tag.getKey().equals("jobId")) jobId = tag.getValue(); if (tag.getKey().equals("Name")) name = tag.getValue(); } - this.projectId = projectId; - this.deploymentId = deploymentId; - this.name = name; state = ec2Instance.getState(); availabilityZone = ec2Instance.getPlacement().getAvailabilityZone(); launchTime = ec2Instance.getLaunchTime(); diff --git a/src/main/java/com/conveyal/datatools/manager/models/Organization.java b/src/main/java/com/conveyal/datatools/manager/models/Organization.java index 381d7cf07..585f311d6 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Organization.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Organization.java @@ -30,8 +30,8 @@ public class Organization extends Model implements Serializable { public boolean active; public UsageTier usageTier; public Set extensions = new HashSet<>(); - public Date subscriptionBeginDate; - public Date subscriptionEndDate; + public long subscriptionBeginDate; + public long subscriptionEndDate; public Organization () {} diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/FeedStore.java b/src/main/java/com/conveyal/datatools/manager/persistence/FeedStore.java index f5dfe21de..08ba90d0a 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/FeedStore.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/FeedStore.java @@ -158,7 +158,7 @@ public Long getFeedSize (String id) { } } - private static AWSCredentialsProvider getAWSCreds () { + public static AWSCredentialsProvider getAWSCreds () { if (S3_CREDENTIALS_FILENAME != null) { return new ProfileCredentialsProvider(S3_CREDENTIALS_FILENAME, "default"); } else { diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java index d7d497671..66a266192 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/Persistence.java @@ -55,6 +55,7 @@ public class Persistence { public static void initialize () { PojoCodecProvider pojoCodecProvider = PojoCodecProvider.builder() + .register("com.conveyal.datatools.manager.jobs") .register("com.conveyal.datatools.manager.models") .register("com.conveyal.gtfs.loader") .register("com.conveyal.gtfs.validator") diff --git a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java index 2b1231322..8d02bbc6d 100644 --- a/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java +++ b/src/main/java/com/conveyal/datatools/manager/persistence/TypedPersistence.java @@ -141,7 +141,7 @@ public List getAll () { * We should really have a bit more abstraction here. */ public List getFiltered (Bson filter) { - return mongoCollection.find(filter).into(new ArrayList()); + return mongoCollection.find(filter).into(new ArrayList<>()); } /** From 18bd9d3cc1ad41c4c389257211b6fb89ae61984e Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 20 Sep 2019 15:26:32 -0400 Subject: [PATCH 33/42] refactor(deploy-to-ec2): add json property latest; add server ID to summary --- .../java/com/conveyal/datatools/manager/jobs/DeployJob.java | 2 ++ .../java/com/conveyal/datatools/manager/models/Deployment.java | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 768d96dce..aeda31e7e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -831,6 +831,7 @@ public static class DeployStatus extends Status { */ public static class DeploySummary implements Serializable { private static final long serialVersionUID = 1L; + public String serverId; public long duration; public String s3Bucket; public String jobId; @@ -845,6 +846,7 @@ public static class DeploySummary implements Serializable { public DeploySummary () { } public DeploySummary (DeployJob job) { + this.serverId = job.otpServer.id; this.ec2Info = job.otpServer.ec2Info; this.otpVersion = job.deployment.otpVersion; this.jobId = job.jobId; diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 91a14b9d5..ca71e368b 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -156,6 +156,7 @@ public Date retrieveLastDeployed () { } /** Get latest deployment summary. */ + @JsonProperty("latest") public DeployJob.DeploySummary latest () { return deployJobSummaries.size() > 0 ? deployJobSummaries.get(0) : null; } From 41c21a80f602da28c81c75e6a166fe702837d2e2 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 20 Sep 2019 16:09:49 -0400 Subject: [PATCH 34/42] refactor(deploy): fix check for S3 jar --- .../datatools/manager/jobs/DeployJob.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index aeda31e7e..9b1c1ea6b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -65,6 +65,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.codec.binary.Base64; +import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -696,11 +697,20 @@ private String constructUserData(boolean graphAlreadyBuilt) { String s3JarBucket = deployment.r5 ? "r5-builds" : "opentripplanner-builds"; String s3JarKey = jarName + ".jar"; // If jar does not exist in bucket, fail job. - if (!FeedStore.s3Client.doesObjectExist(s3JarBucket, s3JarKey)) { - status.fail(String.format("Requested jar does not exist at s3://%s/%s", s3JarBucket, s3JarKey)); + String s3JarUrl = String.format("https://%s.s3.amazonaws.com/%s", s3JarBucket, s3JarKey); + try { + final URL url = new URL(s3JarUrl); + HttpURLConnection huc = (HttpURLConnection) url.openConnection(); + huc.setRequestMethod("HEAD"); + int responseCode = huc.getResponseCode(); + if (responseCode != HttpStatus.OK_200) { + status.fail(String.format("Requested trip planner jar does not exist at s3://%s/%s", s3JarBucket, s3JarKey)); + return null; + } + } catch (IOException e) { + status.fail(String.format("Error checking for trip planner jar: s3://%s/%s", s3JarBucket, s3JarKey)); return null; } - String s3JarUrl = String.format("https://%s.s3.amazonaws.com/%s", s3JarBucket, s3JarKey); String jarDir = String.format("/opt/%s", getTripPlannerString()); List lines = new ArrayList<>(); String routerName = "default"; From 7e1528b91a7e7c907726e416c1c5598c218cfba6 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 24 Sep 2019 11:17:42 -0400 Subject: [PATCH 35/42] refactor(deploy-to-ec2): address PR comments --- .../datatools/common/utils/SparkUtils.java | 2 +- .../controllers/api/DeploymentController.java | 2 +- .../controllers/api/ServerController.java | 5 +++++ .../datatools/manager/jobs/DeployJob.java | 22 ++++++++++++------- .../manager/jobs/MonitorServerStatusJob.java | 4 ++++ 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java index 8f4428cfb..d27ffc5c5 100644 --- a/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java +++ b/src/main/java/com/conveyal/datatools/common/utils/SparkUtils.java @@ -112,7 +112,7 @@ public static void logMessageAndHalt( ) throws HaltException { // Note that halting occurred, also print error stacktrace if applicable if (e != null) e.printStackTrace(); - LOG.info("Halting with status code {}. Error message: {}.", statusCode, message); + LOG.info("Halting with status code {}. Error message: {}", statusCode, message); if (statusCode >= 500) { LOG.error(message); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index d3921f207..0ae6b21c9 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -112,7 +112,7 @@ private static String downloadBuildArtifact (Request req, Response res) { OtpServer server = Persistence.servers.getById(deployment.deployedTo); if (server == null) { uriString = String.format("s3://%s/bundles/%s/%s/%s", "S3_BUCKET", deployment.projectId, deployment.id, jobId); - logMessageAndHalt(req, 400, "Cannot construct URI for build artifact. " + uriString); + logMessageAndHalt(req, 400, "The deployment does not have job history or associated server information to construct URI for build artifact. " + uriString); return null; } uriString = String.format("s3://%s/bundles/%s/%s/%s", server.s3Bucket, deployment.projectId, deployment.id, jobId); diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index c5afa39ff..0dd7f5c57 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -141,6 +141,11 @@ public static TerminateInstancesResult terminateInstances(String... instanceIds) return terminateInstances(Arrays.asList(instanceIds)); } + /** Convenience method to override {@link #terminateInstances(Collection)}. */ + public static TerminateInstancesResult terminateInstances(List instances) throws AmazonEC2Exception { + return terminateInstances(getIds(instances)); + } + /** * Create a new server for the project. All feed sources with a valid latest version are added to the new * deployment. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 9b1c1ea6b..3e7eebd0c 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -452,7 +452,7 @@ private void replaceEC2Servers() { List instances = startEC2Instances(1, false); // Exit if an error was encountered. if (status.error || instances.size() == 0) { - ServerController.terminateInstances(getIds(instances)); + ServerController.terminateInstances(instances); return; } status.message = "Waiting for graph build to complete..."; @@ -474,7 +474,7 @@ private void replaceEC2Servers() { statusMessage = "Error encountered while building graph. Inspect build logs."; LOG.error(statusMessage); status.fail(statusMessage); - ServerController.terminateInstances(getIds(instances)); + ServerController.terminateInstances(instances); return; } // Spin up remaining servers which will download the graph from S3. @@ -486,7 +486,7 @@ private void replaceEC2Servers() { status.message = String.format("Spinning up remaining %d instance(s).", remainingServerCount); remainingInstances.addAll(startEC2Instances(remainingServerCount, true)); if (remainingInstances.size() == 0 || status.error) { - ServerController.terminateInstances(getIds(remainingInstances)); + ServerController.terminateInstances(remainingInstances); return; } // Create new thread pool to monitor server setup so that the servers are monitored in parallel. @@ -639,6 +639,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { .withTags(new Tag("jobId", this.jobId)) .withTags(new Tag("serverId", otpServer.id)) .withTags(new Tag("routerId", getRouterId())) + .withTags(new Tag("user", this.owner)) .withResources(instance.getInstanceId()) ); } @@ -704,11 +705,15 @@ private String constructUserData(boolean graphAlreadyBuilt) { huc.setRequestMethod("HEAD"); int responseCode = huc.getResponseCode(); if (responseCode != HttpStatus.OK_200) { - status.fail(String.format("Requested trip planner jar does not exist at s3://%s/%s", s3JarBucket, s3JarKey)); + statusMessage = String.format("Requested trip planner jar does not exist at s3://%s/%s", s3JarBucket, s3JarKey); + LOG.error(statusMessage); + status.fail(statusMessage); return null; } } catch (IOException e) { - status.fail(String.format("Error checking for trip planner jar: s3://%s/%s", s3JarBucket, s3JarKey)); + statusMessage = String.format("Error checking for trip planner jar: s3://%s/%s", s3JarBucket, s3JarKey); + LOG.error(statusMessage); + status.fail(statusMessage); return null; } String jarDir = String.format("/opt/%s", getTripPlannerString()); @@ -754,15 +759,16 @@ private String constructUserData(boolean graphAlreadyBuilt) { // Build the graph if Graph object (presumably this is the first instance to be started up). if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu java -Xmx6G -jar %s/%s.jar point --build %s", jarDir, jarName, routerDir)); else lines.add(String.format("sudo -H -u ubuntu java -jar %s/%s.jar --build %s > $BUILDLOGFILE 2>&1", jarDir, jarName, routerDir)); - // Upload the graph and build log file to S3. + // Upload the build log file and graph to S3. if (!deployment.r5) { - lines.add(String.format("aws s3 --region us-east-1 cp %s/%s %s ", routerDir, OTP_GRAPH_FILENAME, getS3GraphURI())); String s3BuildLogPath = joinToS3FolderURI(getBuildLogFilename()); lines.add(String.format("aws s3 --region us-east-1 cp $BUILDLOGFILE %s ", s3BuildLogPath)); + lines.add(String.format("aws s3 --region us-east-1 cp %s/%s %s ", routerDir, OTP_GRAPH_FILENAME, getS3GraphURI())); } } - // Upload user data log. + // Get the instance's instance ID from the AWS metadata endpoint. lines.add("instance_id=`curl http://169.254.169.254/latest/meta-data/instance-id`"); + // Upload user data log associated with instance to a log file on S3. lines.add(String.format("aws s3 --region us-east-1 cp $USERDATALOG %s/${instance_id}.log", getS3FolderURI().toString())); if (deployment.buildGraphOnly) { // If building graph only, tell the instance to shut itself down after the graph build (and log upload) is diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 3f7b8c05c..ac07eb6e0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -181,6 +181,10 @@ private void wait(String waitingFor) { } } + /** + * Get the S3 key for the bundle status file, which is uploaded by the graph-building EC2 instance after the graph + * build completes. The file contains either "SUCCESS" or "FAILURE". + */ private String getBundleStatusKey () { return String.join("/", deployJob.getJobRelativePath(), DeployJob.BUNDLE_DOWNLOAD_COMPLETE_FILE); } From f34affbae6d2be2047c8cb540a53dad2bbf9f5bd Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 24 Sep 2019 14:14:44 -0400 Subject: [PATCH 36/42] refactor(deploy-to-ec2): surround s3 checks in try/catch --- .../manager/jobs/MonitorServerStatusJob.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index ac07eb6e0..5069ddb57 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -11,6 +11,7 @@ import com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.S3Object; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.models.Deployment; @@ -168,7 +169,13 @@ private boolean jobHasTimedOut() { private boolean isGraphBuilt() { AmazonS3URI uri = new AmazonS3URI(deployJob.getS3GraphURI()); LOG.info("Checking for graph at {}", uri.toString()); - return FeedStore.s3Client.doesObjectExist(uri.getBucket(), uri.getKey()); + // Surround with try/catch (exception thrown if object does not exist). + try { + return FeedStore.s3Client.doesObjectExist(uri.getBucket(), uri.getKey()); + } catch (AmazonS3Exception e) { + LOG.warn("Object not found for key " + uri.getKey(), e); + return false; + } } /** Have the current thread sleep for a few seconds in order to pause during a while loop. */ @@ -193,7 +200,13 @@ private String getBundleStatusKey () { private boolean isBundleDownloaded() { String key = getBundleStatusKey(); LOG.info("Checking for bundle complete at s3://{}/{}", otpServer.s3Bucket, key); - return FeedStore.s3Client.doesObjectExist(otpServer.s3Bucket, key); + // Surround with try/catch (exception thrown if object does not exist). + try { + return FeedStore.s3Client.doesObjectExist(otpServer.s3Bucket, key); + } catch (AmazonS3Exception e) { + LOG.warn("Object not found for key " + key, e); + return false; + } } /** From 4cc9b68cac9ae1c98df037078ed0a7d643a0576f Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 24 Sep 2019 14:43:29 -0400 Subject: [PATCH 37/42] refactor(deploy-to-ec2): actually skip termination request --- .../datatools/manager/controllers/api/ServerController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index 0dd7f5c57..c0c7d5bbc 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -130,6 +130,7 @@ public static List getIds (List instances) { public static TerminateInstancesResult terminateInstances(Collection instanceIds) throws AmazonEC2Exception { if (instanceIds.size() == 0) { LOG.warn("No instance IDs provided in list. Skipping termination request."); + return null; } LOG.info("Terminating EC2 instances {}", instanceIds); TerminateInstancesRequest request = new TerminateInstancesRequest().withInstanceIds(instanceIds); From fb44a614e56845fe58798c9023cf962ae84162e3 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 30 Sep 2019 14:45:35 -0400 Subject: [PATCH 38/42] refactor(deploy): fix duration calc --- .../java/com/conveyal/datatools/manager/jobs/DeployJob.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 3e7eebd0c..bb46ebd1f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -428,7 +428,7 @@ public void jobFinished () { deployment.deployedTo = otpServer.id; deployment.deployJobSummaries.add(0, new DeploySummary(this)); Persistence.deployments.replace(deployment.id, deployment); - long durationMinutes = TimeUnit.MILLISECONDS.toMinutes(status.duration); + long durationMinutes = TimeUnit.MILLISECONDS.toMinutes(System.currentTimeMillis() - status.startTime); message = String.format("Deployment %s successfully deployed to %s in %s minutes.", deployment.name, otpServer.publicUrl, durationMinutes); } else { message = String.format("WARNING: Deployment %s failed to deploy to %s. Error: %s", deployment.name, otpServer.publicUrl, status.message); @@ -756,13 +756,15 @@ private String constructUserData(boolean graphAlreadyBuilt) { lines.add(String.format("printf \"{\\n bikeRentalFile: \"bikeshare.xml\"\\n}\" >> %s/build-config.json\"", routerDir)); } lines.add("echo 'starting graph build'"); - // Build the graph if Graph object (presumably this is the first instance to be started up). + // Build the graph. if (deployment.r5) lines.add(String.format("sudo -H -u ubuntu java -Xmx6G -jar %s/%s.jar point --build %s", jarDir, jarName, routerDir)); else lines.add(String.format("sudo -H -u ubuntu java -jar %s/%s.jar --build %s > $BUILDLOGFILE 2>&1", jarDir, jarName, routerDir)); // Upload the build log file and graph to S3. if (!deployment.r5) { String s3BuildLogPath = joinToS3FolderURI(getBuildLogFilename()); lines.add(String.format("aws s3 --region us-east-1 cp $BUILDLOGFILE %s ", s3BuildLogPath)); + // FIXME Add check fof graph build file existence +// lines.add(String.format("[ -f %s/%s ] && GRAPH_BUILD_STATUS='SUCCESS' || GRAPH_BUILD_STATUS='FAILURE'", routerDir, OTP_GRAPH_FILENAME)); lines.add(String.format("aws s3 --region us-east-1 cp %s/%s %s ", routerDir, OTP_GRAPH_FILENAME, getS3GraphURI())); } } From 101b7f977e7677e5d45206b67042483a7dd7d7c3 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 1 Oct 2019 18:02:25 -0400 Subject: [PATCH 39/42] refactor(deploy): use onboard nginx to signal ec2 deploy status --- configurations/default/server.yml.tmp | 4 + .../datatools/manager/jobs/DeployJob.java | 64 ++++++++---- .../manager/jobs/MonitorServerStatusJob.java | 99 +++++++++++-------- 3 files changed, 109 insertions(+), 58 deletions(-) diff --git a/configurations/default/server.yml.tmp b/configurations/default/server.yml.tmp index 62df3dd69..760ed9cd8 100644 --- a/configurations/default/server.yml.tmp +++ b/configurations/default/server.yml.tmp @@ -24,6 +24,10 @@ modules: ec2: enabled: false default_ami: ami-your-ami-id + # Note: using a cloudfront URL for these download URLs will greatly + # increase download/deploy speed. + otp_download_url: https://optional-otp-repo.com + r5_download_url: https://optional-r5-repo.com user_admin: enabled: true gtfsapi: diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index bb46ebd1f..96b08134e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -86,9 +86,18 @@ public class DeployJob extends MonitorableJob { private static final String AMI_CONFIG_PATH = "modules.deployment.ec2.default_ami"; private static final String DEFAULT_AMI_ID = DataManager.getConfigPropertyAsText(AMI_CONFIG_PATH); private static final String OTP_GRAPH_FILENAME = "Graph.obj"; - public static final String BUNDLE_DOWNLOAD_COMPLETE_FILE = "BUNDLE_DOWNLOAD_COMPLETE"; + // Use txt at the end of these filenames so that these can easily be viewed in a web browser. + public static final String BUNDLE_DOWNLOAD_COMPLETE_FILE = "BUNDLE_DOWNLOAD_COMPLETE.txt"; + public static final String GRAPH_STATUS_FILE = "GRAPH_STATUS.txt"; private static final long TEN_MINUTES_IN_MILLISECONDS = 10 * 60 * 1000; - /** + // Note: using a cloudfront URL for these download repo URLs will greatly increase download/deploy speed. + private static final String R5_REPO_URL = DataManager.hasConfigProperty("modules.deployment.r5_download_url") + ? DataManager.getConfigPropertyAsText("modules.deployment.r5_download_url") + : "https://r5-builds.s3.amazonaws.com"; + private static final String OTP_REPO_URL = DataManager.hasConfigProperty("modules.deployment.otp_download_url") + ? DataManager.getConfigPropertyAsText("modules.deployment.otp_download_url") + : "https://opentripplanner-builds.s3.amazonaws.com"; + /** * S3 bucket to upload deployment to. If not null, uses {@link OtpServer#s3Bucket}. Otherwise, defaults to * {@link DataManager#feedBucket} * */ @@ -121,6 +130,16 @@ public String getDeploymentId () { return deployment.id; } + /** Increment the completed servers count (for use during ELB deployment) and update the job status. */ + public void incrementCompletedServers() { + status.numServersCompleted++; + int totalServers = otpServer.ec2Info.instanceCount; + if (totalServers < 1) totalServers = 1; + int numRemaining = totalServers - status.numServersCompleted; + double newStatus = status.percentComplete + (100 - status.percentComplete) * numRemaining / totalServers; + status.update(String.format("Completed %d servers. %d remaining...", status.numServersCompleted, numRemaining), newStatus); + } + @JsonProperty public String getServerId () { return otpServer.id; @@ -478,19 +497,19 @@ private void replaceEC2Servers() { return; } // Spin up remaining servers which will download the graph from S3. - int remainingServerCount = otpServer.ec2Info.instanceCount <= 0 ? 0 : otpServer.ec2Info.instanceCount - 1; + status.numServersRemaining = otpServer.ec2Info.instanceCount <= 0 ? 0 : otpServer.ec2Info.instanceCount - 1; List remainingServerMonitorJobs = new ArrayList<>(); List remainingInstances = new ArrayList<>(); - if (remainingServerCount > 0) { + if (status.numServersRemaining > 0) { // Spin up remaining EC2 instances. - status.message = String.format("Spinning up remaining %d instance(s).", remainingServerCount); - remainingInstances.addAll(startEC2Instances(remainingServerCount, true)); + status.message = String.format("Spinning up remaining %d instance(s).", status.numServersRemaining); + remainingInstances.addAll(startEC2Instances(status.numServersRemaining, true)); if (remainingInstances.size() == 0 || status.error) { ServerController.terminateInstances(remainingInstances); return; } // Create new thread pool to monitor server setup so that the servers are monitored in parallel. - ExecutorService service = Executors.newFixedThreadPool(remainingServerCount); + ExecutorService service = Executors.newFixedThreadPool(status.numServersRemaining); for (Instance instance : remainingInstances) { // Note: new instances are added MonitorServerStatusJob monitorServerStatusJob = new MonitorServerStatusJob(owner, this, instance, true); @@ -630,7 +649,7 @@ private List startEC2Instances(int count, boolean graphAlreadyBuilt) { for (Instance instance : instances) { // The public IP addresses will likely be null at this point because they take a few seconds to initialize. instanceIpAddresses.put(instance.getInstanceId(), instance.getPublicIpAddress()); - String serverName = String.format("%s %s (%s) %d", deployment.r5 ? "r5" : "otp", deployment.name, dateString, serverCounter++); + String serverName = String.format("%s %s (%s) %d %s", deployment.r5 ? "r5" : "otp", deployment.name, dateString, serverCounter++, graphAlreadyBuilt ? "clone" : "builder"); LOG.info("Creating tags for new EC2 instance {}", serverName); ec2.createTags(new CreateTagsRequest() .withTags(new Tag("Name", serverName)) @@ -695,24 +714,24 @@ private String constructUserData(boolean graphAlreadyBuilt) { jarName = deployment.r5 ? deployment.r5Version : deployment.otpVersion; Persistence.deployments.replace(deployment.id, deployment); } - String s3JarBucket = deployment.r5 ? "r5-builds" : "opentripplanner-builds"; + // Construct URL for trip planner jar and check that it exists with a lightweight HEAD request. String s3JarKey = jarName + ".jar"; - // If jar does not exist in bucket, fail job. - String s3JarUrl = String.format("https://%s.s3.amazonaws.com/%s", s3JarBucket, s3JarKey); + String repoUrl = deployment.r5 ? R5_REPO_URL : OTP_REPO_URL; + String s3JarUrl = String.join("/", repoUrl, s3JarKey); try { final URL url = new URL(s3JarUrl); HttpURLConnection huc = (HttpURLConnection) url.openConnection(); huc.setRequestMethod("HEAD"); int responseCode = huc.getResponseCode(); if (responseCode != HttpStatus.OK_200) { - statusMessage = String.format("Requested trip planner jar does not exist at s3://%s/%s", s3JarBucket, s3JarKey); + statusMessage = String.format("Requested trip planner jar does not exist at %s", s3JarUrl); LOG.error(statusMessage); status.fail(statusMessage); return null; } } catch (IOException e) { - statusMessage = String.format("Error checking for trip planner jar: s3://%s/%s", s3JarBucket, s3JarKey); - LOG.error(statusMessage); + statusMessage = String.format("Error checking for trip planner jar: %s", s3JarUrl); + LOG.error(statusMessage, e); status.fail(statusMessage); return null; } @@ -735,6 +754,10 @@ private String constructUserData(boolean graphAlreadyBuilt) { lines.add(String.format("rm -rf %s/*", routerDir)); // Download trip planner JAR. lines.add(String.format("mkdir -p %s", jarDir)); + // Add client static file directory for uploading deploy stage status files. + // TODO: switch to AMI that uses /usr/share/nginx/html as static file dir so we don't have to create this new dir. + lines.add("WEB_DIR=/usr/share/nginx/client"); + lines.add("sudo mkdir $WEB_DIR"); lines.add(String.format("wget %s -O %s/%s.jar", s3JarUrl, jarDir, jarName)); if (graphAlreadyBuilt) { lines.add("echo 'downloading graph from s3'"); @@ -745,9 +768,8 @@ private String constructUserData(boolean graphAlreadyBuilt) { lines.add(String.format("aws s3 --region us-east-1 cp %s /tmp/bundle.zip", getS3BundleURI())); // Determine if bundle download was successful. lines.add("[ -f /tmp/bundle.zip ] && BUNDLE_STATUS='SUCCESS' || BUNDLE_STATUS='FAILURE'"); - // Create and upload file with bundle status to notify Data Tools that download is complete. - lines.add(String.format("echo $BUNDLE_STATUS > /tmp/%s", BUNDLE_DOWNLOAD_COMPLETE_FILE)); - lines.add(String.format("aws s3 --region us-east-1 cp /tmp/%s %s", BUNDLE_DOWNLOAD_COMPLETE_FILE, joinToS3FolderURI(BUNDLE_DOWNLOAD_COMPLETE_FILE))); + // Create file with bundle status in web dir to notify Data Tools that download is complete. + lines.add(String.format("sudo echo $BUNDLE_STATUS > $WEB_DIR/%s", BUNDLE_DOWNLOAD_COMPLETE_FILE)); // Put unzipped bundle data into router directory. lines.add(String.format("unzip /tmp/bundle.zip -d %s", routerDir)); // FIXME: Add ability to fetch custom bikeshare.xml file (CarFreeAtoZ) @@ -763,11 +785,13 @@ private String constructUserData(boolean graphAlreadyBuilt) { if (!deployment.r5) { String s3BuildLogPath = joinToS3FolderURI(getBuildLogFilename()); lines.add(String.format("aws s3 --region us-east-1 cp $BUILDLOGFILE %s ", s3BuildLogPath)); - // FIXME Add check fof graph build file existence -// lines.add(String.format("[ -f %s/%s ] && GRAPH_BUILD_STATUS='SUCCESS' || GRAPH_BUILD_STATUS='FAILURE'", routerDir, OTP_GRAPH_FILENAME)); lines.add(String.format("aws s3 --region us-east-1 cp %s/%s %s ", routerDir, OTP_GRAPH_FILENAME, getS3GraphURI())); } } + // Determine if graph build/download was successful. + lines.add(String.format("[ -f %s/%s ] && GRAPH_STATUS='SUCCESS' || GRAPH_STATUS='FAILURE'", routerDir, OTP_GRAPH_FILENAME)); + // Create file with bundle status in web dir to notify Data Tools that download is complete. + lines.add(String.format("sudo echo $GRAPH_STATUS > $WEB_DIR/%s", GRAPH_STATUS_FILE)); // Get the instance's instance ID from the AWS metadata endpoint. lines.add("instance_id=`curl http://169.254.169.254/latest/meta-data/instance-id`"); // Upload user data log associated with instance to a log file on S3. @@ -835,6 +859,8 @@ public static class DeployStatus extends Status { /** To how many servers have we successfully deployed thus far? */ public int numServersCompleted; + public int numServersRemaining; + /** How many servers are we attempting to deploy to? */ public int totalServers; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 5069ddb57..7afcd4a4a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -12,7 +12,6 @@ import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.S3Object; import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.OtpServer; @@ -29,6 +28,9 @@ import java.io.IOException; +import static com.conveyal.datatools.manager.jobs.DeployJob.BUNDLE_DOWNLOAD_COMPLETE_FILE; +import static com.conveyal.datatools.manager.jobs.DeployJob.GRAPH_STATUS_FILE; + /** * Job that is dispatched during a {@link DeployJob} that spins up EC2 instances. This handles waiting for the server to * come online and for the OTP application/API to become available. @@ -75,11 +77,13 @@ public String getDeploymentId () { @Override public void jobLogic() { + String message; + String ipUrl = "http://" + instance.getPublicIpAddress(); // Get OTP URL for instance to check for availability. - boolean routerIsAvailable = false; + boolean routerIsAvailable = false, graphIsAvailable = false; // If graph was not already built by a previous server, wait for it to build. if (!graphAlreadyBuilt) { - boolean bundleIsDownloaded = false, graphBuildIsComplete = false; + boolean bundleIsDownloaded = false; // Progressively check status of OTP server if (deployment.buildGraphOnly) { // No need to check that OTP is running. Just check to see that the graph is built. @@ -88,52 +92,70 @@ public void jobLogic() { } // First, check that OTP has started up. status.update("Prepping for graph build...", 20); + String bundleUrl = String.join("/", ipUrl, BUNDLE_DOWNLOAD_COMPLETE_FILE); + long bundleDownloadStartTime = System.currentTimeMillis(); while (!bundleIsDownloaded) { // If the request is successful, the OTP instance has started. - wait("bundle download check"); - bundleIsDownloaded = isBundleDownloaded(); + wait("bundle download check:" + bundleUrl); + bundleIsDownloaded = checkForSuccessfulRequest(bundleUrl); if (jobHasTimedOut()) { status.fail(String.format("Job timed out while checking for server bundle download status (%s)", instance.getInstanceId())); return; } } // Check status of bundle download and fail job if there was a failure. - String bundleStatus = FeedStore.s3Client.getObjectAsString(otpServer.s3Bucket, getBundleStatusKey()); + String bundleStatus = getUrlAsString(bundleUrl); if (bundleStatus == null || !bundleStatus.contains("SUCCESS")) { status.fail("Failure encountered while downloading transit bundle."); return; } - status.update("Building graph...", 30); - long graphBuildStartTime = System.currentTimeMillis(); - while (!graphBuildIsComplete) { - // If the request is successful, the OTP instance has started. - wait("graph build check"); - graphBuildIsComplete = isGraphBuilt(); - if (jobHasTimedOut()) { - status.fail(String.format("Job timed out while waiting for graph build (%s)", instance.getInstanceId())); - return; - } - } - graphBuildSeconds = (System.currentTimeMillis() - graphBuildStartTime) / 1000; - String message = String.format("Graph build completed in %d seconds!", graphBuildSeconds); + long bundleDownloadSeconds = (System.currentTimeMillis() - bundleDownloadStartTime) / 1000; + message = String.format("Bundle downloaded in %d seconds!", bundleDownloadSeconds); LOG.info(message); - if (deployment.buildGraphOnly) { - status.update(false, message, 100); + status.update("Building graph...", 30); + } + status.update("Loading graph...", 40); + long graphBuildStartTime = System.currentTimeMillis(); + String graphStatusUrl = String.join("/", ipUrl, GRAPH_STATUS_FILE); + while (!graphIsAvailable) { + // If the request is successful, the OTP instance has started. + wait("graph build/download check: " + graphStatusUrl); + graphIsAvailable = checkForSuccessfulRequest(graphStatusUrl); + if (jobHasTimedOut()) { + message = String.format("Job timed out while waiting for graph build/download (%s)", instance.getInstanceId()); + LOG.error(message); + status.fail(message); return; } } - status.update("Loading graph...", 40); + // Check status of bundle download and fail job if there was a failure. + String graphStatus = getUrlAsString(graphStatusUrl); + if (graphStatus == null || !graphStatus.contains("SUCCESS")) { + message = String.format("Failure encountered while building/downloading graph (%s).", instance.getInstanceId()); + LOG.error(message); + status.fail(message); + return; + } + graphBuildSeconds = (System.currentTimeMillis() - graphBuildStartTime) / 1000; + message = String.format("Graph build/download completed in %d seconds!", graphBuildSeconds); + LOG.info(message); + // If only task is to build graph, this machine's job is complete and we can consider this job done. + if (deployment.buildGraphOnly) { + status.update(false, message, 100); + return; + } // Once this is confirmed, check for the existence of the router, which will indicate that the graph build is // complete. - String routerUrl = String.format("http://%s/otp/routers/default", instance.getPublicIpAddress()); + String routerUrl = String.join("/", ipUrl, "otp/routers/default"); while (!routerIsAvailable) { - LOG.info("Checking that router is available (i.e., graph build or read is finished) at {}", routerUrl); // If the request was successful, the graph build is complete! // TODO: Substitute in specific router ID? Or just default to... default. - wait("trip planner to start up"); + wait("router to become available: " + routerUrl); routerIsAvailable = checkForSuccessfulRequest(routerUrl); if (jobHasTimedOut()) { - status.fail(String.format("Job timed out while waiting for trip planner to start up (%s)", instance.getInstanceId())); + message = String.format("Job timed out while waiting for trip planner to start up (%s)", instance.getInstanceId()); + status.fail(message); + LOG.error(message); return; } } @@ -147,11 +169,12 @@ public void jobLogic() { .withTargets(new TargetDescription().withId(instance.getInstanceId())); elbClient.registerTargets(registerTargetsRequest); // FIXME how do we know it was successful? - String message = String.format("Server successfully registered with load balancer %s. OTP running at %s", otpServer.ec2Info.targetGroupArn, routerUrl); + message = String.format("Server successfully registered with load balancer %s. OTP running at %s", otpServer.ec2Info.targetGroupArn, routerUrl); LOG.info(message); status.update(false, message, 100, true); + deployJob.incrementCompletedServers(); } else { - String message = String.format("There is no load balancer under which to register ec2 instance %s.", instance.getInstanceId()); + message = String.format("There is no load balancer under which to register ec2 instance %s.", instance.getInstanceId()); LOG.error(message); status.fail(message); } @@ -193,19 +216,17 @@ private void wait(String waitingFor) { * build completes. The file contains either "SUCCESS" or "FAILURE". */ private String getBundleStatusKey () { - return String.join("/", deployJob.getJobRelativePath(), DeployJob.BUNDLE_DOWNLOAD_COMPLETE_FILE); + return String.join("/", deployJob.getJobRelativePath(), BUNDLE_DOWNLOAD_COMPLETE_FILE); } - /** Check if the bundle download completed file has been uploaded to S3. */ - private boolean isBundleDownloaded() { - String key = getBundleStatusKey(); - LOG.info("Checking for bundle complete at s3://{}/{}", otpServer.s3Bucket, key); - // Surround with try/catch (exception thrown if object does not exist). - try { - return FeedStore.s3Client.doesObjectExist(otpServer.s3Bucket, key); - } catch (AmazonS3Exception e) { - LOG.warn("Object not found for key " + key, e); - return false; + private String getUrlAsString(String url) { + HttpGet httpGet = new HttpGet(url); + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + return EntityUtils.toString(response.getEntity()); + } catch (IOException e) { + LOG.error("Could not complete request to {}", url); + e.printStackTrace(); + return null; } } From 523801dc702943faf71b2d2c24e9cc07b12c1926 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Thu, 3 Oct 2019 11:34:40 -0400 Subject: [PATCH 40/42] refactor(deploy): bump default otp version to 1.4 --- .../java/com/conveyal/datatools/manager/models/Deployment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index ca71e368b..c6b939af6 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -59,7 +59,7 @@ public class Deployment extends Model implements Serializable { public String name; - public static final String DEFAULT_OTP_VERSION = "otp-v1.3.0"; + public static final String DEFAULT_OTP_VERSION = "otp-v1.4.0"; public static final String DEFAULT_R5_VERSION = "v2.4.1-9-g3be6daa"; /** What server is this currently deployed to? */ From c399189b00ed2affd42db912a1445330f462b237 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 8 Oct 2019 11:08:13 -0400 Subject: [PATCH 41/42] refactor(deploy): add terminate EC2 instance HTTP endpoint --- .../controllers/api/DeploymentController.java | 74 +++++++++++++++---- 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 0ae6b21c9..ecd3e6f25 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -2,6 +2,7 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.AmazonEC2Exception; import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.DescribeInstancesResult; import com.amazonaws.services.ec2.model.Filter; @@ -22,7 +23,6 @@ import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.json.JsonManager; -import com.mongodb.client.FindIterable; import org.bson.Document; import org.eclipse.jetty.http.HttpStatus; import org.slf4j.Logger; @@ -63,7 +63,7 @@ public class DeploymentController { * Gets the deployment specified by the request's id parameter and ensure that user has access to the * deployment. If the user does not have permission the Spark request is halted with an error. */ - private static Deployment checkDeploymentPermissions (Request req, Response res) { + private static Deployment getDeploymentWithPermissions(Request req, Response res) { Auth0UserProfile userProfile = req.attribute("user"); String deploymentId = req.params("id"); Deployment deployment = Persistence.deployments.getById(deploymentId); @@ -79,11 +79,11 @@ private static Deployment checkDeploymentPermissions (Request req, Response res) } private static Deployment getDeployment (Request req, Response res) { - return checkDeploymentPermissions(req, res); + return getDeploymentWithPermissions(req, res); } private static Deployment deleteDeployment (Request req, Response res) { - Deployment deployment = checkDeploymentPermissions(req, res); + Deployment deployment = getDeploymentWithPermissions(req, res); deployment.delete(); return deployment; } @@ -92,7 +92,7 @@ private static Deployment deleteDeployment (Request req, Response res) { * HTTP endpoint for downloading a build artifact (e.g., otp build log or Graph.obj) from S3. */ private static String downloadBuildArtifact (Request req, Response res) { - Deployment deployment = checkDeploymentPermissions(req, res); + Deployment deployment = getDeploymentWithPermissions(req, res); DeployJob.DeploySummary summaryToDownload = null; String uriString = null; // If a jobId query param is provided, find the matching job summary. @@ -134,7 +134,7 @@ private static String downloadBuildArtifact (Request req, Response res) { * TODO: Should there be an option to download the OSM network as well? */ private static FileInputStream downloadDeployment (Request req, Response res) throws IOException { - Deployment deployment = checkDeploymentPermissions(req, res); + Deployment deployment = getDeploymentWithPermissions(req, res); // Create temp file in order to generate input stream. File temp = File.createTempFile("deployment", ".zip"); // just include GTFS, not any of the ancillary information @@ -251,8 +251,8 @@ private static Deployment createDeploymentFromFeedSource (Request req, Response * Update a single deployment. If the deployment's feed versions are updated, checks to ensure that each * version exists and is a part of the same parent project are performed before updating. */ - private static Object updateDeployment (Request req, Response res) { - Deployment deploymentToUpdate = checkDeploymentPermissions(req, res); + private static Deployment updateDeployment (Request req, Response res) { + Deployment deploymentToUpdate = getDeploymentWithPermissions(req, res); Document updateDocument = Document.parse(req.body()); // FIXME use generic update hook, also feedVersions is getting serialized into MongoDB (which is undesirable) // Check that feed versions in request body are OK to add to deployment, i.e., they exist and are a part of @@ -295,11 +295,60 @@ private static Object updateDeployment (Request req, Response res) { return updatedDeployment; } + // TODO: Add some point it may be useful to refactor DeployJob to allow adding an EC2 instance to an existing job, + // but for now that can be achieved by using the AWS EC2 console: choose an EC2 instance to replicate and select + // "Run more like this". Then follow the prompts to replicate the instance. +// private static Object addEC2InstanceToDeployment(Request req, Response res) { +// Deployment deployment = getDeploymentWithPermissions(req, res); +// List currentEC2Instances = deployment.retrieveEC2Instances(); +// EC2InstanceSummary ec2ToClone = currentEC2Instances.get(0); +// RunInstancesRequest request = new RunInstancesRequest(); +// ec2.runInstances() +// ec2ToClone. +// DeployJob.DeploySummary latestDeployJob = deployment.latest(); +// +// } + + /** + * HTTP endpoint to terminate a set of instance IDs that are associated with a particular deployment. The intent here + * is to give the user a device by which they can terminate an EC2 instance that has started up, but is not responding + * or otherwise failed to successfully become an OTP instance as part of an ELB deployment (or perhaps two people + * somehow kicked off a deploy job for the same deployment simultaneously and one of the EC2 instances has + * out-of-date data). + */ + private static boolean terminateEC2InstanceForDeployment(Request req, Response res) { + Deployment deployment = getDeploymentWithPermissions(req, res); + String instanceIds = req.queryParams("instanceIds"); + if (instanceIds == null) { + logMessageAndHalt(req, 400, "Must provide one or more instance IDs."); + return false; + } + String[] idsToTerminate = instanceIds.split(","); + // Ensure that request does not contain instance IDs which are not associated with this deployment. + List ec2InstancesForDeployment = deployment.retrieveEC2Instances().stream() + .map(ec2InstanceSummary -> ec2InstanceSummary.instanceId) + .collect(Collectors.toList()); + for (String id : idsToTerminate) { + if (!ec2InstancesForDeployment.contains(id)) { + logMessageAndHalt(req, HttpStatus.UNAUTHORIZED_401, "It is not permitted to terminate an instance that is not associated with deployment " + deployment.id); + return false; + } + } + // If checks are ok, terminate instances. + try { + ServerController.terminateInstances(idsToTerminate); + } catch (AmazonEC2Exception e) { + logMessageAndHalt(req, 400, "Could not complete termination request", e); + return false; + } + return true; + } + /** * HTTP controller to fetch information about provided EC2 machines that power ELBs running a trip planner. */ private static List fetchEC2InstanceSummaries(Request req, Response res) { - Deployment deployment = checkDeploymentPermissions(req, res); + Deployment deployment = getDeploymentWithPermissions(req, res); return deployment.retrieveEC2Instances(); } @@ -332,7 +381,7 @@ private static String deploy (Request req, Response res) { // Check parameters supplied in request for validity. Auth0UserProfile userProfile = req.attribute("user"); String target = req.params("target"); - Deployment deployment = checkDeploymentPermissions(req, res); + Deployment deployment = getDeploymentWithPermissions(req, res); Project project = Persistence.projects.getById(deployment.projectId); if (project == null) logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); @@ -402,13 +451,12 @@ public static void register (String apiPrefix) { get(apiPrefix + "secure/deployments/:id/download", DeploymentController::downloadDeployment); get(apiPrefix + "secure/deployments/:id/artifact", DeploymentController::downloadBuildArtifact); get(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::fetchEC2InstanceSummaries, json::write); - // TODO: In the future, we may have need for terminating a single EC2 instance. For now, an admin using the AWS - // console should suffice. -// delete(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::terminateEC2Instance, json::write); + delete(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::terminateEC2InstanceForDeployment, json::write); get(apiPrefix + "secure/deployments/:id", DeploymentController::getDeployment, json::write); delete(apiPrefix + "secure/deployments/:id", DeploymentController::deleteDeployment, json::write); get(apiPrefix + "secure/deployments", DeploymentController::getAllDeployments, json::write); post(apiPrefix + "secure/deployments", DeploymentController::createDeployment, json::write); +// post(apiPrefix + "secure/deployments/:id/ec2", DeploymentController::addEC2InstanceToDeployment, json::write); put(apiPrefix + "secure/deployments/:id", DeploymentController::updateDeployment, json::write); post(apiPrefix + "secure/deployments/fromfeedsource/:id", DeploymentController::createDeploymentFromFeedSource, json::write); } From 240a6e012d789d81af9e28ca8839b9ec96725ee6 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 8 Oct 2019 15:50:35 -0400 Subject: [PATCH 42/42] refactor(deploy): refine terminate instances endpoint and check for graph size during deploy --- .../controllers/api/DeploymentController.java | 41 +++++++++++++------ .../controllers/api/ServerController.java | 26 ++++++++++++ .../datatools/manager/jobs/DeployJob.java | 36 ++++------------ 3 files changed, 61 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index ecd3e6f25..dafd81ba1 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -2,7 +2,6 @@ import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; -import com.amazonaws.services.ec2.model.AmazonEC2Exception; import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.DescribeInstancesResult; import com.amazonaws.services.ec2.model.Filter; @@ -34,6 +33,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; @@ -310,11 +310,11 @@ private static Deployment updateDeployment (Request req, Response res) { // } /** - * HTTP endpoint to terminate a set of instance IDs that are associated with a particular deployment. The intent here - * is to give the user a device by which they can terminate an EC2 instance that has started up, but is not responding - * or otherwise failed to successfully become an OTP instance as part of an ELB deployment (or perhaps two people - * somehow kicked off a deploy job for the same deployment simultaneously and one of the EC2 instances has - * out-of-date data). + * HTTP endpoint to deregister and terminate a set of instance IDs that are associated with a particular deployment. + * The intent here is to give the user a device by which they can terminate an EC2 instance that has started up, but + * is not responding or otherwise failed to successfully become an OTP instance as part of an ELB deployment (or + * perhaps two people somehow kicked off a deploy job for the same deployment simultaneously and one of the EC2 + * instances has out-of-date data). */ private static boolean terminateEC2InstanceForDeployment(Request req, Response res) { Deployment deployment = getDeploymentWithPermissions(req, res); @@ -323,22 +323,37 @@ private static boolean terminateEC2InstanceForDeployment(Request req, Response r logMessageAndHalt(req, 400, "Must provide one or more instance IDs."); return false; } - String[] idsToTerminate = instanceIds.split(","); + List idsToTerminate = Arrays.asList(instanceIds.split(",")); // Ensure that request does not contain instance IDs which are not associated with this deployment. - List ec2InstancesForDeployment = deployment.retrieveEC2Instances().stream() + List instances = deployment.retrieveEC2Instances(); + List instanceIdsForDeployment = instances.stream() .map(ec2InstanceSummary -> ec2InstanceSummary.instanceId) .collect(Collectors.toList()); + // Get the target group ARN from the latest deployment. Surround in a try/catch in case of NPEs. + // TODO: Perhaps provide some other way to provide the target group ARN. + String targetGroupArn; + try { + targetGroupArn = deployment.latest().ec2Info.targetGroupArn; + } catch (Exception e) { + logMessageAndHalt(req, 400, "Latest deploy job does not exist or is missing target group ARN."); + return false; + } for (String id : idsToTerminate) { - if (!ec2InstancesForDeployment.contains(id)) { + if (!instanceIdsForDeployment.contains(id)) { logMessageAndHalt(req, HttpStatus.UNAUTHORIZED_401, "It is not permitted to terminate an instance that is not associated with deployment " + deployment.id); return false; } + int code = instances.get(instanceIdsForDeployment.indexOf(id)).state.getCode(); + // 48 indicates instance is terminated, 32 indicates shutting down. Prohibit terminating an already + if (code == 48 || code == 32) { + logMessageAndHalt(req, 400, "Instance is already terminated/shutting down: " + id); + return false; + } } // If checks are ok, terminate instances. - try { - ServerController.terminateInstances(idsToTerminate); - } catch (AmazonEC2Exception e) { - logMessageAndHalt(req, 400, "Could not complete termination request", e); + boolean success = ServerController.deRegisterAndTerminateInstances(targetGroupArn, idsToTerminate); + if (!success) { + logMessageAndHalt(req, 400, "Could not complete termination request"); return false; } return true; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java index c0c7d5bbc..93d13ba33 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ServerController.java @@ -21,7 +21,9 @@ import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancing; import com.amazonaws.services.elasticloadbalancingv2.AmazonElasticLoadBalancingClient; import com.amazonaws.services.elasticloadbalancingv2.model.AmazonElasticLoadBalancingException; +import com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest; import com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetGroupsRequest; +import com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription; import com.amazonaws.services.elasticloadbalancingv2.model.TargetGroup; import com.amazonaws.services.identitymanagement.AmazonIdentityManagement; import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClientBuilder; @@ -30,6 +32,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.jobs.MonitorServerStatusJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.JsonViews; import com.conveyal.datatools.manager.models.OtpServer; @@ -147,6 +150,29 @@ public static TerminateInstancesResult terminateInstances(List instanc return terminateInstances(getIds(instances)); } + /** + * De-register instances from the specified target group/load balancer and terminate the instances. + * + */ + public static boolean deRegisterAndTerminateInstances(String targetGroupArn, List instanceIds) { + LOG.info("De-registering instances from load balancer {}", instanceIds); + TargetDescription[] targetDescriptions = instanceIds.stream() + .map(id -> new TargetDescription().withId(id)) + .toArray(TargetDescription[]::new); + try { + DeregisterTargetsRequest request = new DeregisterTargetsRequest() + .withTargetGroupArn(targetGroupArn) + .withTargets(targetDescriptions); + AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); + elb.deregisterTargets(request); + ServerController.terminateInstances(instanceIds); + } catch (AmazonEC2Exception | AmazonElasticLoadBalancingException e) { + LOG.warn("Could not terminate EC2 instances: " + String.join(",", instanceIds), e); + return false; + } + return true; + } + /** * Create a new server for the project. All feed sources with a valid latest version are added to the new * deployment. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index 96b08134e..119b2d38f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -539,7 +539,7 @@ private void replaceEC2Servers() { .map(instance -> instance.instanceId) .collect(Collectors.toList()); if (previousInstanceIds.size() > 0) { - boolean success = deRegisterAndTerminateInstances(previousInstanceIds); + boolean success = ServerController.deRegisterAndTerminateInstances(otpServer.ec2Info.targetGroupArn, previousInstanceIds); // If there was a problem during de-registration/termination, notify via status message. if (!success) { finalMessage = String.format("Server setup is complete! (WARNING: Could not terminate previous EC2 instances: %s", previousInstanceIds); @@ -553,30 +553,6 @@ private void replaceEC2Servers() { } } - /** - * De-register instances from the load balancer and terminate the instanced. - * - * (Note: new instances are registered with load balancer in {@link MonitorServerStatusJob}.) - */ - private boolean deRegisterAndTerminateInstances(List instanceIds) { - LOG.info("De-registering instances from load balancer {}", instanceIds); - TargetDescription[] targetDescriptions = instanceIds.stream() - .map(id -> new TargetDescription().withId(id)) - .toArray(TargetDescription[]::new); - DeregisterTargetsRequest deregisterTargetsRequest = new DeregisterTargetsRequest() - .withTargetGroupArn(otpServer.ec2Info.targetGroupArn) - .withTargets(targetDescriptions); - AmazonElasticLoadBalancing elb = AmazonElasticLoadBalancingClient.builder().build(); - elb.deregisterTargets(deregisterTargetsRequest); - try { - ServerController.terminateInstances(instanceIds); - } catch (AmazonEC2Exception e) { - LOG.warn("Could not terminate EC2 instances {}", instanceIds); - return false; - } - return true; - } - /** * Start the specified number of EC2 instances based on the {@link OtpServer#ec2Info}. * @param count number of EC2 instances to start @@ -739,6 +715,7 @@ private String constructUserData(boolean graphAlreadyBuilt) { List lines = new ArrayList<>(); String routerName = "default"; String routerDir = String.format("/var/%s/graphs/%s", getTripPlannerString(), routerName); + String graphPath = String.join("/", routerDir, OTP_GRAPH_FILENAME); //////////////// BEGIN USER DATA lines.add("#!/bin/bash"); // Send trip planner logs to LOGFILE @@ -762,7 +739,7 @@ private String constructUserData(boolean graphAlreadyBuilt) { if (graphAlreadyBuilt) { lines.add("echo 'downloading graph from s3'"); // Download Graph from S3. - lines.add(String.format("aws s3 --region us-east-1 cp %s %s/%s ", getS3GraphURI(), routerDir, OTP_GRAPH_FILENAME)); + lines.add(String.format("aws s3 --region us-east-1 cp %s %s ", getS3GraphURI(), graphPath)); } else { // Download data bundle from S3. lines.add(String.format("aws s3 --region us-east-1 cp %s /tmp/bundle.zip", getS3BundleURI())); @@ -785,11 +762,12 @@ private String constructUserData(boolean graphAlreadyBuilt) { if (!deployment.r5) { String s3BuildLogPath = joinToS3FolderURI(getBuildLogFilename()); lines.add(String.format("aws s3 --region us-east-1 cp $BUILDLOGFILE %s ", s3BuildLogPath)); - lines.add(String.format("aws s3 --region us-east-1 cp %s/%s %s ", routerDir, OTP_GRAPH_FILENAME, getS3GraphURI())); + lines.add(String.format("aws s3 --region us-east-1 cp %s %s ", graphPath, getS3GraphURI())); } } - // Determine if graph build/download was successful. - lines.add(String.format("[ -f %s/%s ] && GRAPH_STATUS='SUCCESS' || GRAPH_STATUS='FAILURE'", routerDir, OTP_GRAPH_FILENAME)); + // Determine if graph build/download was successful (and that Graph.obj is not zero bytes). + lines.add(String.format("FILESIZE=$(wc -c <%s)", graphPath)); + lines.add(String.format("[ -f %s ] && (($FILESIZE > 0)) && GRAPH_STATUS='SUCCESS' || GRAPH_STATUS='FAILURE'", graphPath)); // Create file with bundle status in web dir to notify Data Tools that download is complete. lines.add(String.format("sudo echo $GRAPH_STATUS > $WEB_DIR/%s", GRAPH_STATUS_FILE)); // Get the instance's instance ID from the AWS metadata endpoint.