Fleet-up cleanup (allianceauth#798)

Add caching and better error handling
Move fleetup templates into fleetup app
Move fleetup urls into fleetup app
Fix button overflow and url
Add manager unit tests
Fix python3 compatibility

alliance_auth/settings.py.example

@@ -57,6 +57,7 @@ INSTALLED_APPS = [
     'optimer',
     'corputils',
     'fleetactivitytracking',
+    'fleetup',
     'notifications',
     'esi',
     'permissions_tool',
@@ -661,6 +662,10 @@ LOGGING = {
             'handlers': ['log_file', 'console', 'notifications'],
             'level': 'ERROR',
         },
+        'fleetup': {
+            'handlers': ['log_file', 'console', 'notifications'],
+            'level': 'DEBUG',
+        },
         'util': {
             'handlers': ['log_file', 'console', 'notifications'],
             'level': 'DEBUG',

alliance_auth/tests/test_settings.py

@@ -14,6 +14,7 @@
 NOSE_ARGS = [
     #'--with-coverage',
     #'--cover-package=',
+    #'--exe',  # If your tests need this to be found/run, check they py files are not chmodded +x
 ]
 
 # Celery configuration
@@ -48,6 +49,7 @@
     'optimer',
     'corputils',
     'fleetactivitytracking',
+    'fleetup',
     'notifications',
     'esi',
     'permissions_tool',

alliance_auth/urls.py

@@ -11,7 +11,7 @@
 import optimer.views
 import timerboard.views
 import fleetactivitytracking.views
-import fleetup.views
+import fleetup.urls
 import srp.views
 import notifications.views
 import hrapplications.views
@@ -75,12 +75,7 @@
 urlpatterns += i18n_patterns(
 
     # Fleetup
-    url(r'^fleetup/$', fleetup.views.fleetup_view, name='auth_fleetup_view'),
-    url(r'^fleetup/fittings/$', fleetup.views.fleetup_fittings, name='auth_fleetup_fittings'),
-    url(r'^fleetup/fittings/(?P<fittingnumber>[0-9]+)/$', fleetup.views.fleetup_fitting, name='auth_fleetup_fitting'),
-    url(r'^fleetup/doctrines/$', fleetup.views.fleetup_doctrines, name='auth_fleetup_doctrines'),
-    url(r'^fleetup/characters/$', fleetup.views.fleetup_characters, name='auth_fleetup_characters'),
-    url(r'^fleetup/doctrines/(?P<doctrinenumber>[0-9]+)/$', fleetup.views.fleetup_doctrine, name='auth_fleetup_doctrine'),
+    url(r'^fleetup/', include(fleetup.urls.urlpatterns)),
 
     # Authentication
     url(_(r'^login_user/'), authentication.views.login_user, name='auth_login_user'),

fleetup/__init__.py

@@ -1 +1,2 @@
 from __future__ import unicode_literals
+default_app_config = 'fleetup.apps.FleetupConfig'

fleetup/managers.py

@@ -1,178 +1,184 @@
 from __future__ import unicode_literals
 from django.conf import settings
+from django.core.cache import cache
 from datetime import datetime
 
 import logging
 import requests
-import json
+import hashlib
 
 logger = logging.getLogger(__name__)
 
-appkey = settings.FLEETUP_APP_KEY
-userid = settings.FLEETUP_USER_ID
-apiid = settings.FLEETUP_API_ID
-groupid = settings.FLEETUP_GROUP_ID
-
 
 class FleetUpManager:
+    APP_KEY = settings.FLEETUP_APP_KEY
+    USER_ID = settings.FLEETUP_USER_ID
+    API_ID = settings.FLEETUP_API_ID
+    GROUP_ID = settings.FLEETUP_GROUP_ID
+    BASE_URL = "http://api.fleet-up.com/Api.svc/{}/{}/{}".format(APP_KEY, USER_ID, API_ID)
+
     def __init__(self):
         pass
 
-    @staticmethod
-    def get_fleetup_members():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/GroupCharacters/" + str(groupid) + ""
+    @classmethod
+    def _request_cache_key(cls, url):
+        h = hashlib.sha1()
+        h.update(url.encode('utf-8'))
+        return 'FLEETUP_ENDPOINT_' + h.hexdigest()
+
+    @classmethod
+    def _cache_until_seconds(cls, cache_until_json):
+        # Format comes in like "/Date(1493896236163)/"
         try:
-            jsondata = requests.get(url).content
-            fmembers = json.loads(jsondata.decode())
+            epoch_ms = int(cache_until_json[6:-2])
+            cache_delta = datetime.fromtimestamp(epoch_ms/1000) - datetime.now()
+            cache_delta_seconds = cache_delta.total_seconds()
+            if cache_delta_seconds < 0:
+                return 0
+            elif cache_delta_seconds > 3600:
+                return 3600
+            else:
+                return cache_delta_seconds
+        except TypeError:
+            logger.debug("Couldn't convert CachedUntil time, defaulting to 600 seconds")
+        return 600
+
+    @classmethod
+    def get_endpoint(cls, url):
+        try:
+            cache_key = cls._request_cache_key(url)
+            cached = cache.get(cache_key)
+            if cached:
+                return cached
+
+            r = requests.get(url)
+            r.raise_for_status()
+
+            json = r.json()
+
+            if json['Success']:
+                cache.set(cache_key, json, cls._cache_until_seconds(json['CachedUntilUTC']))
+            return json
+        except requests.exceptions.ConnectionError:
+            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
+        except requests.HTTPError:
+            logger.exception("Error accessing Fleetup API")
+        return None
+
+    @classmethod
+    def get_fleetup_members(cls):
+        url = "{}/GroupCharacters/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        try:
+            fmembers = cls.get_endpoint(url)
+            if not fmembers:
+                return None
             return {row["UserId"]: {"user_id": row["UserId"],
                                     "char_name": row["EveCharName"],
                                     "char_id": row["EveCharId"],
                                     "corporation": row["Corporation"]} for row in fmembers["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
         except (ValueError, UnicodeDecodeError, TypeError):
             logger.debug("No fleetup members retrieved.")
         return {}
 
-    @staticmethod
-    def get_fleetup_operations():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Operations/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            foperations = json.loads(jsondata.decode())
-            return {row["StartString"]: {"subject": row["Subject"],
-                                         "start": (datetime.strptime(row["StartString"], "%Y-%m-%d %H:%M:%S")),
-                                         "end": (datetime.strptime(row["EndString"], "%Y-%m-%d %H:%M:%S")),
-                                         "operation_id": row["OperationId"],
-                                         "location": row["Location"],
-                                         "location_info": row["LocationInfo"],
-                                         "details": row["Details"],
-                                         "url": row["Url"],
-                                         "doctrine": row["Doctrines"],
-                                         "organizer": row["Organizer"]} for row in foperations["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.debug("No fleetup operations retrieved.")
-        return {}
+    @classmethod
+    def get_fleetup_operations(cls):
+        url = "{}/Operations/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        foperations = cls.get_endpoint(url)
+        if foperations is None:
+            return None
+        return {row["StartString"]: {"subject": row["Subject"],
+                                     "start": datetime.strptime(row["StartString"], "%Y-%m-%d %H:%M:%S"),
+                                     "end": datetime.strptime(row["EndString"], "%Y-%m-%d %H:%M:%S"),
+                                     "operation_id": row["OperationId"],
+                                     "location": row["Location"],
+                                     "location_info": row["LocationInfo"],
+                                     "details": row["Details"],
+                                     "url": row["Url"],
+                                     "doctrine": row["Doctrines"],
+                                     "organizer": row["Organizer"]} for row in foperations["Data"]}
 
-    @staticmethod
-    def get_fleetup_timers():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Timers/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            ftimers = json.loads(jsondata.decode())
-            return {row["ExpiresString"]: {"solarsystem": row["SolarSystem"],
-                                           "planet": row["Planet"],
-                                           "moon": row["Moon"],
-                                           "owner": row["Owner"],
-                                           "type": row["Type"],
-                                           "timer_type": row["TimerType"],
-                                           "expires": (datetime.strptime(row["ExpiresString"], "%Y-%m-%d %H:%M:%S")),
-                                           "notes": row["Notes"]} for row in ftimers["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError, TypeError):
-            logger.debug("No fleetup timers retrieved.")
+    @classmethod
+    def get_fleetup_timers(cls):
+        url = "{}/Timers/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        ftimers = cls.get_endpoint(url)
+        if not ftimers:
+            return None
+        return {row["ExpiresString"]: {"solarsystem": row["SolarSystem"],
+                                       "planet": row["Planet"],
+                                       "moon": row["Moon"],
+                                       "owner": row["Owner"],
+                                       "type": row["Type"],
+                                       "timer_type": row["TimerType"],
+                                       "expires": (datetime.strptime(row["ExpiresString"], "%Y-%m-%d %H:%M:%S")),
+                                       "notes": row["Notes"]} for row in ftimers["Data"]}
         return {}
 
-    @staticmethod
-    def get_fleetup_doctrines():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Doctrines/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            fdoctrines = json.loads(jsondata.decode())
-            return {"fleetup_doctrines": fdoctrines["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.debug("No fleetup doctrines retrieved.")
-        return {"fleetup_doctrines": []}
-
-    @staticmethod
-    def get_fleetup_doctrine(doctrinenumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/DoctrineFittings/%s" % doctrinenumber
-        try:
-            jsondata = requests.get(url).content
-            fdoctrine = json.loads(jsondata.decode())
-            return {"fitting_doctrine": fdoctrine}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup doctrine number %s not found" % doctrinenumber)
-        return {"fitting_doctrine": {}}
-
-    @staticmethod
-    def get_fleetup_fittings():
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fittings/" + str(groupid) + ""
-        try:
-            jsondata = requests.get(url).content
-            ffittings = json.loads(jsondata.decode())
-            return {row["FittingId"]: {"fitting_id": row["FittingId"],
-                                       "name": row["Name"],
-                                       "icon_id": row["EveTypeId"],
-                                       "hull": row["HullType"],
-                                       "shiptype": row["ShipType"],
-                                       "estimated": row["EstPrice"],
-                                       "faction": row["Faction"],
-                                       "categories": row["Categories"],
-                                       "last_update": (
-                                       datetime.strptime(row["LastUpdatedString"], "%Y-%m-%d %H:%M:%S"))} for row in
-                    ffittings["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError, TypeError):
-            logger.debug("No fleetup fittings retrieved.")
-        return {}
+    @classmethod
+    def get_fleetup_doctrines(cls):
+        url = "{}/Doctrines/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        fdoctrines = cls.get_endpoint(url)
+        if not fdoctrines:
+            return None
+        return {"fleetup_doctrines": fdoctrines["Data"]}
 
-    @staticmethod
-    def get_fleetup_fitting(fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fitting/%s" % fittingnumber
+    @classmethod
+    def get_fleetup_doctrine(cls, doctrinenumber):
+        url = "{}/DoctrineFittings/{}".format(cls.BASE_URL, doctrinenumber)
+        fdoctrine = cls.get_endpoint(url)
+        if not fdoctrine:
+            return None
+        return {"fitting_doctrine": fdoctrine}
+
+    @classmethod
+    def get_fleetup_fittings(cls):
+        url = "{}/Fittings/{}".format(cls.BASE_URL, cls.GROUP_ID)
+        ffittings = cls.get_endpoint(url)
+        if not ffittings:
+            return None
+        return {row["FittingId"]: {"fitting_id": row["FittingId"],
+                                   "name": row["Name"],
+                                   "icon_id": row["EveTypeId"],
+                                   "hull": row["HullType"],
+                                   "shiptype": row["ShipType"],
+                                   "estimated": row["EstPrice"],
+                                   "faction": row["Faction"],
+                                   "categories": row["Categories"],
+                                   "last_update": (
+                                   datetime.strptime(row["LastUpdatedString"], "%Y-%m-%d %H:%M:%S"))} for row in
+                ffittings["Data"]}
+
+    @classmethod
+    def get_fleetup_fitting(cls, fittingnumber):
+        url = "{}/Fitting/{}".format(cls.BASE_URL, fittingnumber)
         try:
-            jsondata = requests.get(url).content
-            ffitting = json.loads(jsondata.decode())
+            ffitting = cls.get_endpoint(url)
+            if not ffitting:
+                return None
             return {"fitting_data": ffitting["Data"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup fitting number %s not found" % fittingnumber)
         except KeyError:
             logger.warn("Failed to retrieve fleetup fitting number %s" % fittingnumber)
         return {"fitting_data": {}}
 
-    @staticmethod
-    def get_fleetup_doctrineid(fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fitting/%s" % fittingnumber
+    @classmethod
+    def get_fleetup_doctrineid(cls, fittingnumber):
+        url = "{}/Fitting/{}".format(cls.BASE_URL, fittingnumber)
         try:
-            jsondata = requests.get(url).content
-            fdoctrineid = json.loads(jsondata.decode())
+            fdoctrineid = cls.get_endpoint(url)
+            if not fdoctrineid:
+                return None
             return fdoctrineid['Data']['Doctrines'][0]['DoctrineId']
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
-            logger.warn("Fleetup doctrine number not found for fitting number %s" % fittingnumber)
         except (KeyError, IndexError):
             logger.debug("Fleetup fitting number %s not in a doctrine." % fittingnumber)
-        return None
+        return {}
 
-    @staticmethod
-    def get_fleetup_fitting_eft(fittingnumber):
-        url = "http://api.fleet-up.com/Api.svc/" + str(appkey) + "/" + str(userid) + "/" + str(
-            apiid) + "/Fitting/%s/eft" % fittingnumber
+    @classmethod
+    def get_fleetup_fitting_eft(cls, fittingnumber):
+        url = "{}/Fitting/{}/eft".format(cls.BASE_URL, fittingnumber)
         try:
-            jsondata = requests.get(url).content
-            ffittingeft = json.loads(jsondata.decode())
+            ffittingeft = cls.get_endpoint(url)
+            if not ffittingeft:
+                return None
             return {"fitting_eft": ffittingeft["Data"]["FittingData"]}
-        except requests.exceptions.ConnectionError:
-            logger.warn("Can't connect to Fleet-Up API, is it offline?!")
-        except (ValueError, UnicodeDecodeError):
+        except KeyError:
             logger.warn("Fleetup fitting eft not found for fitting number %s" % fittingnumber)
         return {"fitting_eft": {}}