Merge branch 'v4.0.0' into new-ui

.gitignore

@@ -3,3 +3,4 @@ export
 logs
 uploads
 *.pyc
+

GTFSManager.py

@@ -31,6 +31,7 @@
 import numpy as np
 import io # used in hyd csv import
 import requests, platform # used to log user stats
+requests.packages.urllib3.disable_warnings() # suppress warning messages like "InsecureRequestWarning: Unverified HTTPS request is being made." from https://stackoverflow.com/a/44850849/4355695
 
 # setting constants
 root = os.path.dirname(__file__) # needed for tornado
@@ -47,6 +48,8 @@
 configFile = 'config.json'
 thisURL = ''
 
+# paths were you must not tread.. see "MyStaticFileHandler" class in apy.py
+forbiddenPaths = ['/pw/']
 debugMode = False # using this flag at various places to do or not do things based on whether we're in development or production
 
 requiredFeeds = ['agency.txt','calendar.txt','stops.txt','routes.txt','trips.txt','stop_times.txt']
@@ -73,7 +76,23 @@
 
 logmessage('Loaded dependencies, starting static GTFS Manager program.')
 
-'''
+#################################
+# Tornado classes : One class = one API call / request
+
+# 22.4.19 Making a custom class/function to control how the user's browser loads normal URLs
+# from https://stackoverflow.com/a/55762431/4355695 : restrict direct browser access to .py files and stuff
+class MyStaticFileHandler(tornado.web.StaticFileHandler):
+    def validate_absolute_path(self, root, absolute_path):
+        if absolute_path.endswith('.py') or any([ (x in absolute_path) for x in forbiddenPaths]):
+            # raise tornado.web.HTTPError(403) # this is the usual thing: raise 403 Forbidden error. But instead..
+            return os.path.join(root,'lib','errorpage.txt')
+
+        if absolute_path.endswith('favicon.ico'):
+            return os.path.join(root,'lib','favicon.ico')
+
+        return super().validate_absolute_path(root, absolute_path) # you may pass
+
+comment= '''
 # Tornado API functions template:
 class APIHandler(tornado.web.RequestHandler):
 	def get(self):
@@ -85,9 +104,9 @@ def get(self):
 
 	def post(self):
 		user = self.get_argument("username")
-		passwd = self.get_argument("password")
+		data = json.loads( self.request.body.decode('UTF-8') )
 		time.sleep(10)
-		self.write("Your username is %s and password is %s" % (user, passwd))
+		self.write("data:",data)
 '''
 
 class allStops(tornado.web.RequestHandler):
@@ -1464,7 +1483,9 @@ def make_app():
 		(r"/API/Config/ApiKeys", APIKeys),
 		(r"/API/gtfs/shapes", gtfsshape),
 		#(r"/API/idList", idList),
-		(r"/(.*)", tornado.web.StaticFileHandler, {"path": root, "default_filename": "index.html"})
+		# (r"/(.*)", tornado.web.StaticFileHandler, {"path": root, "default_filename": "index.html"})
+        (r"/(.*)", MyStaticFileHandler, {"path": root, "default_filename": "index.html"})
+
 	])
 
 # for catching Ctrl+C and exiting gracefully. From https://nattster.wordpress.com/2013/06/05/catch-kill-signal-in-python/

GTFSserverfunctions.py

@@ -248,7 +248,7 @@ def importGTFS(zipname):
 				# print('last ID: ' + IDList[-1])
 				workChunk = chunk[ chunk[IDcol].isin(IDList[:-1]) ]
 				if len(carryOverChunk):
-					workChunk = pd.concat([carryOverChunk, workChunk],ignore_index=True)
+					workChunk = pd.concat([carryOverChunk, workChunk],ignore_index=True, sort=False)
 				carryOverChunk = chunk[ chunk[IDcol] == IDList[-1] ]
 
 				fileCounter += 1
@@ -532,8 +532,7 @@ def replaceTableDB(tablename, data, key=None, value=None):
 			logmessage('Note: {} does not have any data.'.format(h5File))
 			oldLen = 0
 
-
-		df3 = pd.concat([df,xdf], ignore_index=True)
+		df3 = pd.concat([df,xdf], ignore_index=True, sort=False)
 		df3.to_hdf(dbFolder+h5File, 'df', format='table', mode='w', complevel=1)
 
 		logmessage('Replaced {} entries for {}={} with {} new entries in {}.'\
@@ -1299,7 +1298,6 @@ def replaceChunkyTableDB(xdf, value, tablename='stop_times'):
 			logmessage('Note: {} does not exist yet, so we will likely create it.'.format(chunkFile))
 
 	# next 3 lines to be done in either case
-	# newdf = pd.concat([df,xdf],ignore_index=True)
 	newdf = df.append(xdf, ignore_index=True, sort=False)
 	logmessage('{} new entries for id {} added. Now writing to {}.'.format( str( len(xdf) ),value, chunkFile ))
 	newdf.to_hdf(dbFolder+chunkFile, 'df', format='table', mode='w', complevel=1)
@@ -1462,7 +1460,7 @@ def readChunkTableDB(tablename, key, value):
 		del df
 
 	if len(collect): 
-		collectDF = pd.concat(collect, ignore_index=True)
+		collectDF = pd.concat(collect, ignore_index=True, sort=False)
 		logmessage('readChunkTableDB: Collected {} rows from table {} for {}={}'\
 			.format(len(collectDF),tablename,key,value),\
 			'\nChunks having entries:',chunksHaving)
@@ -1486,7 +1484,7 @@ def deleteID(column,value):
 		message = 'deleteID: Deleting {} trips first under {}="{}"'.format(len(tripsList),column,value) 
 		logmessage(message)
 		content += message + '<br>'
-		content += ''.join([deleteID('trip_id',trip_id) for trip_id in tripsList]) + '<br>'
+		content += ''.join([deleteID('trip_id',trip_id) for trip_id in tripsList]) + '<br>' # recursive funtion call
 
 	# load deleteRules csv from config folder
 	deleteRulesDF = pd.read_csv(configFolder + 'deleteRules.csv', dtype=str).fillna('')
@@ -1525,6 +1523,12 @@ def deleteInTable(tablename, key, value, action="delete"):
 		if key == chunkRules[tablename].get('key'):
 			h5Files = [findChunk(value, tablename)]
 
+			# intervention : if no chunks found, skip. Ref: https://github.com/WRI-Cities/static-GTFS-manager/issues/150
+			if h5Files == [None]:
+				returnMessage = '{}={} is not present in any {} chunk.<br>'.format(key,value,tablename)
+				logmessage(returnMessage)
+				return returnMessage
+
 			# delete it in the lookup json too.
 			lookupJSONFile = chunkRules[tablename]['lookup']
 			with open(dbFolder + lookupJSONFile) as f:
@@ -1669,7 +1673,7 @@ def logUse(action='launch'):
 		cvar['3'] = ['version', platform.release() ]
 	payload['_cvar'] = json.dumps(cvar)
 	try:
-		r = requests.get('http://nikhilvj.co.in/tracking/piwik.php', params=payload, verify=False, timeout=1)
+		r = requests.get('https://nikhilvj.co.in/tracking/piwik.php', params=payload, verify=False, timeout=1)
 	except requests.exceptions.RequestException as e:
 		# print('exception',e)
-		pass
+		pass

README.md

@@ -5,7 +5,7 @@
 
 A browser-based user interface for creating, editing, exporting of static GTFS (General Transit Feed Specification Reference) feeds for a public transit authority.
 
-**Release Status** : V 3.4.2 (live branch may be some commits ahead)
+**Release Status** : V 3.4.4 (live branch may be some commits ahead)
 Download from [Releases page](https://github.com/WRI-Cities/static-GTFS-manager/releases/).  
 - Windows binary is available in the repo itself, just double-click GTFSManager.exe to start the program.
 - Mac, Ubuntu users can run by first installing docker, then double-clicking two scripts. See [Running with Docker on any OS](https://github.com/WRI-Cities/static-GTFS-manager/wiki/Running-with-Docker-on-any-OS)
@@ -20,10 +20,12 @@ The GTFS data pre-loaded in the program is of Kochi Metro, Kerala, India which o
 
 See the [KMRL open data portal](https://kochimetro.org/open-data/) and some news articles: [1](http://www.newindianexpress.com/cities/kochi/2018/mar/17/kochi-metro-adopts-open-data-system-to-improve-access-to-its-services-1788342.html), [2](http://indianexpress.com/article/india/kochi-metro-throws-open-transit-data-to-public-on-the-lines-of-london-new-york-5100381/), [3](http://www.thehindu.com/news/cities/Kochi/open-data-to-improve-commuter-experience/article23275844.ece), [4](http://www.thehindu.com/news/cities/Kochi/kmrl-moves-a-step-ahead-to-open-up-transit-data/article23247617.ece).
 
-This program adheres to the static GTFS (General Transit Feed Specification Reference) open transit data specs as published by Google Transit here: <https://developers.google.com/transit/gtfs/reference/>  
-It also implements a [GTFS extension for translations](https://developers.google.com/transit/gtfs/reference/gtfs-extensions#translations) of stops and routes names to facilitate multilingual use of the data.
+This program adheres strictly to the static GTFS (General Transit Feed Specification Reference) open transit data specs as published by Google Transit here: <https://developers.google.com/transit/gtfs/reference/>  
 
-Lead programmer up till November 2018: [Nikhil VJ](https://answerquest.github.io) from Pune, India.
+Note: We have created a second complimentary application **[Payanam](https://github.com/WRI-Cities/payanam)** to address needs where a) the system is simple and routes don't have variations; b) properly mapped stops data is not available to begin with.  
+**[Payanam](https://github.com/WRI-Cities/payanam)** can take you from nothing up to a basic GTFS, and after that you can use this application to edit it. So check it out!
+
+Lead programmer as of July 2019: [Nikhil VJ](https://nikhilvj.co.in) from Pune, India.
 
 See this and many more GTFS related resources listed on **[Awesome Transit](https://github.com/CUTR-at-USF/awesome-transit#gtfs-tools)**, a one-stop community listing for all things GTFS.
 

config/settings.js

@@ -22,13 +22,15 @@ xmlhttp.send(null);
 
 // from commonfuncs.js
 
+
 // this flag tells whether it is mandatory for all UIDs to be in capitals or not.
 const CAPSLOCK = false;
 
 // const route_type_options = {0:"0-Tram, Streetcar, Light rail", 1:"1-Subway, Metro", 2:"2-Rail", 3:"3-Bus",4:"4-Ferry", 1100:"1100-Air Service",  };
 // //const route_type_lookup = {0:"Tram, Streetcar, Light rail", 1:"Subway, Metro", 2:"Rail", 3:"Bus",4:"Ferry" };
 // const route_type_lookup = route_type_options;
 
+
 // this json holds the different pages. If you want to add/remove/rename a page, do so here.
 const menu = {
 	"Home": "index.html",

js/deleteID.js

@@ -412,6 +412,7 @@ function populateLists() {
 	globalIDs['stop_id_list'].forEach(function(row){
 		select2items.push({id : row, text: 'stop: ' + row});
 	});
+
 	$("#stop2Delete").select2({				
 		placeholder: "Pick a stop",
 		theme: 'bootstrap4',
@@ -423,6 +424,7 @@ function populateLists() {
 			return;
 		}
 		let stop_id = valueSelected;
+
 		console.log(stop_id);
 		globalKey = 'stop_id';
 		globalValue = stop_id;
@@ -435,6 +437,7 @@ function populateLists() {
 	globalIDs['route_id_list'].forEach(function(row){		
 		select2items.push({id : row, text: 'route: ' + row});
 	});
+
 	$("#route2Delete").select2({				
 		placeholder: "Pick a route",
 		theme: 'bootstrap4',
@@ -446,6 +449,7 @@ function populateLists() {
 			return;
 		}
 		console.log(valueSelected);
+
 		globalKey = 'route_id';
 		globalValue = valueSelected;
 		diagnoseID(globalKey,globalValue);
@@ -457,6 +461,7 @@ function populateLists() {
 	globalIDs['trip_id_list'].forEach(function(row){		
 		select2items.push({id : row, text: 'trip: ' + row});
 	});
+
 	$("#trip2Delete").select2({				
 		placeholder: "Pick a Trip",
 		theme: 'bootstrap4',
@@ -468,6 +473,7 @@ function populateLists() {
 			return;
 		}
 		console.log(valueSelected);
+
 		globalKey = 'trip_id';
 		globalValue = valueSelected;
 		diagnoseID(globalKey,globalValue);
@@ -480,6 +486,7 @@ function populateLists() {
 		select2items.push({id : row, text: 'shape: ' + row});
 
 	});
+
 	$("#shape2Delete").select2({				
 		placeholder: "Pick a Shape",
 		theme: 'bootstrap4',
@@ -491,6 +498,7 @@ function populateLists() {
 			return;
 		}
 		console.log(valueSelected);
+
 		globalKey = 'shape_id';
 		globalValue = valueSelected;
 		diagnoseID(globalKey,globalValue);
@@ -503,6 +511,7 @@ function populateLists() {
 		select2items.push({id : row, text: 'service: ' + row});
 
 	});
+
 	$("#service2Delete").select2({				
 		placeholder: "Pick a Calendar Service",
 		theme: 'bootstrap4',
@@ -514,6 +523,7 @@ function populateLists() {
 			return;
 		}
 		console.log(valueSelected);
+
 		globalKey = 'service_id';
 		globalValue = valueSelected;
 		diagnoseID(globalKey,globalValue);
@@ -526,6 +536,7 @@ function populateLists() {
 	globalIDs['zone_id_list'].forEach(function(row){
 		select2items.push({id : row, text: 'zone: ' + row});
 	});
+
 	$("#zone2Delete").select2({				
 		placeholder: "Pick a Fare Zone",
 		theme: 'bootstrap4',
@@ -537,6 +548,7 @@ function populateLists() {
 			return;
 		}
 		console.log(valueSelected);
+
 		globalKey = 'zone_id';
 		globalValue = valueSelected;
 		diagnoseID(globalKey,globalValue);
@@ -548,6 +560,7 @@ function populateLists() {
 	globalIDs['fare_id_list'].forEach(function(row){
 		select2items.push({id : row, text: 'fareID: ' + row});
 	});
+
 	$("#fareID2Delete").select2({				
 		placeholder: "Pick a Fare ID",
 		theme: 'bootstrap4',
@@ -559,6 +572,7 @@ function populateLists() {
 			return;
 		}
 		console.log(valueSelected);
+
 		globalKey = 'fare_id';
 		globalValue = valueSelected;
 		diagnoseID(globalKey,globalValue);
@@ -570,6 +584,7 @@ function populateLists() {
 	globalIDs['agency_id_list'].forEach(function(row){
 		select2items.push({id : row, text: 'agency: ' + row});
 	});
+
 	$("#agency2Delete").select2({				
 		placeholder: "Pick an Agency ID",
 		theme: 'bootstrap4',
@@ -581,6 +596,7 @@ function populateLists() {
 			return;
 		}
 		console.log(valueSelected);
+
 		globalKey = 'agency_id';
 		globalValue = valueSelected;
 		diagnoseID(globalKey,globalValue);
@@ -637,7 +653,7 @@ function deleteByKey() {
 	console.log(key,value);
 
 	if( ! (key.length && value.length ) ) {
-		$('#deepActionsStatus').html('All values have not been properly set. Please check and try again.');
+		$('#deepActionsStatus').html('<div class="alert alert-danger">All values have not been properly set. Please check and try again.</div>');
 		shakeIt('deepActionsButton'); return;
 	}
 	var jqxhr = $.get( `${APIpath}deleteByKey?pw=${pw}&key=${key}&value=${value}`, function( returndata ) {
@@ -861,4 +877,4 @@ function getPythonTranslations() {
 	};
 	xhr.send();
 }
-*/
+*/

js/homepage.js

@@ -83,7 +83,7 @@ function getPythonPastCommits() {
 
 function exportGTFS() {
 	// lowercase and zap everything that is not a-z, 0-9, - or _  from https://stackoverflow.com/a/4460306/4355695
-	var commit = $("#commitName").val().toLowerCase().replace(/[^a-z0-9-_]/g, "");
+	var commit = $("#commitName").val().toLowerCase().replace(/[^a-z0-9-_.]/g, "");
 
 	$("#commitName").val(commit); // showing the corrected name to user.
 
@@ -248,4 +248,4 @@ function gtfsBlankSlate() {
 			  });			
 		}
 	});
-}
+}

js/sequence.js

@@ -122,11 +122,13 @@ var sequence1table = new Tabulator("#sequence-1-table", {
 
 // #################################
 /* Initiate map */
+
 var LayerOSM0 = L.tileLayer.provider('OpenStreetMap.Mapnik');
 var LayerOSM1 = L.tileLayer.provider('OpenStreetMap.Mapnik');
 
 var mapOptions0 = { 'center': [0, 0], 'zoom': 2, 'layers': LayerOSM0, scrollWheelZoom: false };
 var mapOptions1 = { 'center': [0, 0], 'zoom': 2, 'layers': LayerOSM1, scrollWheelZoom: false };
+
 //var mapOptionsClone = jQuery.extend(true, {}, mapOptions);
 
 var map = [];
@@ -1188,3 +1190,4 @@ function getPythonRoutes() {
 	};
 	xhr.send();
 }
+