Merge pull request #1622 from grycap/stats

Implements: #1353

IM/InfrastructureList.py

@@ -250,7 +250,7 @@ def _save_data_to_db(db_url, inf_list, inf_id=None):
             return None
 
     @staticmethod
-    def _gen_where_from_auth(auth):
+    def _gen_where_from_auth(auth, deleted=0):
         like = ""
         if auth:
             for elem in auth.getAuthInfo('InfrastructureManager'):
@@ -260,12 +260,12 @@ def _gen_where_from_auth(auth):
                     like += "auth like '%%\"" + elem.get("username") + "\"%%'"
 
         if like:
-            return "where deleted = 0 and (" + like + ")"
+            return "where deleted = %d and (%s)" % (deleted, like)
         else:
-            return "where deleted = 0"
+            return "where deleted = %d" % deleted
 
     @staticmethod
-    def _gen_filter_from_auth(auth):
+    def _gen_filter_from_auth(auth, deleted=0):
         like = ""
         if auth:
             for elem in auth.getAuthInfo('InfrastructureManager'):
@@ -275,9 +275,9 @@ def _gen_filter_from_auth(auth):
                     like += '"%s"' % elem.get("username")
 
         if like:
-            return {"deleted": 0, "auth": {"$regex": like}}
+            return {"deleted": deleted, "auth": {"$regex": like}}
         else:
-            return {"deleted": 0}
+            return {"deleted": deleted}
 
     @staticmethod
     def _get_inf_ids_from_db(auth=None):

IM/InfrastructureManager.py

@@ -42,6 +42,7 @@
 from IM.openid.JWT import JWT
 from IM.openid.OpenIDClient import OpenIDClient
 from IM.vault import VaultCredentials
+from IM.Stats import Stats
 
 
 if Config.MAX_SIMULTANEOUS_LAUNCHES > 1:
@@ -2036,3 +2037,21 @@ def EstimateResouces(radl_data, auth):
                         cont += 1
 
         return res
+
+    @staticmethod
+    def GetStats(init_date, end_date, auth):
+        """
+        Get the statistics from the IM DB.
+        Args:
+        - init_date(str): Only will be returned infrastructure created afther this date.
+        - end_date(str): Only will be returned infrastructure created before this date.
+        - auth(Authentication): parsed authentication tokens.
+        Return: a list of dict with the stats.
+        """
+        # First check the auth data
+        auth = InfrastructureManager.check_auth_data(auth)
+        stats = Stats.get_stats(init_date, end_date, auth)
+        if not stats:
+            raise Exception("ERROR connecting with the database!.")
+        else:
+            return stats

IM/REST.py

@@ -21,6 +21,7 @@
 import flask
 import os
 import yaml
+import datetime
 
 from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher
 from cheroot.ssl.builtin import BuiltinSSLAdapter
@@ -1077,6 +1078,49 @@ def RESTChangeInfrastructureAuth(infid=None):
         return return_error(400, "Error modifying infrastructure owner: %s" % get_ex_error(ex))
 
 
+@app.route('/stats', methods=['GET'])
+def RESTGetStats():
+    try:
+        auth = get_auth_header()
+    except Exception:
+        return return_error(401, "No authentication data provided")
+
+    try:
+        init_date = None
+        if "init_date" in flask.request.args.keys():
+            init_date = flask.request.args.get("init_date").lower()
+            init_date = init_date.replace("/", "-")
+            parts = init_date.split("-")
+            try:
+                year = int(parts[0])
+                month = int(parts[1])
+                day = int(parts[2])
+                datetime.date(year, month, day)
+            except Exception:
+                return return_error(400, "Incorrect format in init_date parameter: YYYY/MM/dd")
+        else:
+            init_date = "1970-01-01"
+
+        end_date = None
+        if "end_date" in flask.request.args.keys():
+            end_date = flask.request.args.get("end_date").lower()
+            end_date = end_date.replace("/", "-")
+            parts = end_date.split("-")
+            try:
+                year = int(parts[0])
+                month = int(parts[1])
+                day = int(parts[2])
+                datetime.date(year, month, day)
+            except Exception:
+                return return_error(400, "Incorrect format in end_date parameter: YYYY/MM/dd")
+
+        stats = InfrastructureManager.GetStats(init_date, end_date, auth)
+        return format_output(stats, default_type="application/json", field_name="stats")
+    except Exception as ex:
+        logger.exception("Error getting stats")
+        return return_error(400, "Error getting stats: %s" % get_ex_error(ex))
+
+
 @app.errorhandler(403)
 def error_mesage_403(error):
     return return_error(403, error.description)

IM/ServiceRequests.py

@@ -60,6 +60,7 @@ class IMBaseRequest(AsyncRequest):
     CHANGE_INFRASTRUCTURE_AUTH = "ChangeInfrastructureAuth"
     GET_INFRASTRUCTURE_OWNERS = "GetInfrastructureOwners"
     ESTIMATE_RESOURCES = "EstimateResouces"
+    GET_STATS = "GetStats"
 
     @staticmethod
     def create_request(function, arguments=()):
@@ -119,6 +120,8 @@ def create_request(function, arguments=()):
             return Request_GetInfrastructureOwners(arguments)
         elif function == IMBaseRequest.ESTIMATE_RESOURCES:
             return Request_EstimateResouces(arguments)
+        elif function == IMBaseRequest.GET_STATS:
+            return Request_GetStats(arguments)
         else:
             raise NotImplementedError("Function not Implemented")
 
@@ -473,3 +476,14 @@ def _call_function(self):
         self._error_mesage = "Error getting the resources estimation"
         (radl_data, auth_data) = self.arguments
         return IM.InfrastructureManager.InfrastructureManager.EstimateResouces(radl_data, Authentication(auth_data))
+
+
+class Request_GetStats(IMBaseRequest):
+    """
+    Request class for the GetStats function
+    """
+
+    def _call_function(self):
+        self._error_mesage = "Error getting stats"
+        (init_date, end_date, auth_data) = self.arguments
+        return IM.InfrastructureManager.InfrastructureManager.GetStats(init_date, end_date, Authentication(auth_data))

IM/Stats.py

@@ -0,0 +1,148 @@
+# IM - Infrastructure Manager
+# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+import datetime
+import json
+import yaml
+import logging
+
+from IM.db import DataBase
+from IM.auth import Authentication
+from IM.config import Config
+from IM.InfrastructureList import InfrastructureList
+from radl.radl_parse import parse_radl
+
+
+class Stats():
+
+    logger = logging.getLogger('InfrastructureManager')
+    """Logger object."""
+
+    @staticmethod
+    def _get_data(str_data, init_date=None, end_date=None):
+        dic = json.loads(str_data)
+        resp = {'creation_date': None}
+        if 'creation_date' in dic and dic['creation_date']:
+            creation_date = datetime.datetime.fromtimestamp(float(dic['creation_date']))
+            resp['creation_date'] = str(creation_date)
+            if init_date and creation_date < init_date:
+                return None
+            if end_date and creation_date > end_date:
+                return None
+
+        resp['tosca_name'] = None
+        if 'extra_info' in dic and dic['extra_info'] and "TOSCA" in dic['extra_info']:
+            try:
+                tosca = yaml.safe_load(dic['extra_info']['TOSCA'])
+                icon = tosca.get("metadata", {}).get("icon", "")
+                resp['tosca_name'] = os.path.basename(icon)[:-4]
+            except Exception:
+                Stats.logger.exception("Error loading TOSCA.")
+
+        resp['vm_count'] = 0
+        resp['cpu_count'] = 0
+        resp['memory_size'] = 0
+        resp['cloud_type'] = None
+        resp['cloud_host'] = None
+        resp['hybrid'] = False
+        resp['deleted'] = True if 'deleted' in dic and dic['deleted'] else False
+        for str_vm_data in dic['vm_list']:
+            vm_data = json.loads(str_vm_data)
+            cloud_data = json.loads(vm_data["cloud"])
+
+            # only get the cloud of the first VM
+            if not resp['cloud_type']:
+                resp['cloud_type'] = cloud_data["type"]
+            if not resp['cloud_host']:
+                resp['cloud_host'] = cloud_data["server"]
+            elif resp['cloud_host'] != cloud_data["server"]:
+                resp['hybrid'] = True
+
+            vm_sys = parse_radl(vm_data['info']).systems[0]
+            if vm_sys.getValue('cpu.count'):
+                resp['cpu_count'] += vm_sys.getValue('cpu.count')
+            if vm_sys.getValue('memory.size'):
+                resp['memory_size'] += vm_sys.getFeature('memory.size').getValue('M')
+            resp['vm_count'] += 1
+
+        inf_auth = Authentication.deserialize(dic['auth']).getAuthInfo('InfrastructureManager')[0]
+        resp['im_user'] = inf_auth.get('username')
+        return resp
+
+    @staticmethod
+    def get_stats(init_date="1970-01-01", end_date=None, auth=None):
+        """
+        Get the statistics from the IM DB.
+
+        Args:
+
+        - init_date(str): Only will be returned infrastructure created afther this date.
+        - end_date(str): Only will be returned infrastructure created afther this date.
+        - auth(Authentication): parsed authentication tokens.
+
+        Return: a list of dict with the stats with the following format:
+            {'creation_date': '2022-03-07 13:16:14',
+             'tosca_name': 'kubernetes',
+             'vm_count': 2,
+             'cpu_count': 4,
+             'memory_size': 1024,
+             'cloud_type': 'OSCAR',
+             'cloud_host': 'sharp-elbakyan5.im.grycap.net',
+             'hybrid': False,
+             'im_user': '__OPENID__mcaballer',
+             'inf_id': '1',
+             'deleted': False,
+             'last_date': '2022-03-23'}
+        """
+        stats = []
+        db = DataBase(Config.DATA_DB)
+        if db.connect():
+            if db.db_type == DataBase.MONGO:
+                filt = InfrastructureList._gen_filter_from_auth(auth, 1)
+                if end_date:
+                    filt["date"] = {"$lte": end_date}
+                res = db.find("inf_list", filt, {"id": True, "data": True, "date": True}, [('id', -1)])
+            else:
+                where = InfrastructureList._gen_where_from_auth(auth, 1)
+                if end_date:
+                    where += " and date <= '%s'" % end_date
+                res = db.select("select data, date, id from inf_list %s order by rowid desc" % where)
+
+            for elem in res:
+                if db.db_type == DataBase.MONGO:
+                    data = elem["data"]
+                    date = elem["date"]
+                    inf_id = elem["id"]
+                else:
+                    data = elem[0]
+                    date = elem[1]
+                    inf_id = elem[2]
+                try:
+                    init = datetime.datetime.strptime(init_date, "%Y-%m-%d")
+                    end = datetime.datetime.strptime(end_date, "%Y-%m-%d") if end_date else None
+                    res = Stats._get_data(data, init, end)
+                    if res:
+                        res['inf_id'] = inf_id
+                        res['last_date'] = str(date)
+                        stats.append(res)
+                except Exception:
+                    Stats.logger.exception("ERROR reading infrastructure info from Inf ID: %s" % inf_id)
+            db.close()
+            return stats
+        else:
+            Stats.logger.error("ERROR connecting with the database!.")
+            return None

IM/im_service.py

@@ -229,6 +229,12 @@ def EstimateResources(radl_data, auth_data):
     return WaitRequest(request)
 
 
+def GetStats(init_date, end_date, auth_data):
+    request = IMBaseRequest.create_request(
+        IMBaseRequest.GET_STATS, (init_date, end_date, auth_data))
+    return WaitRequest(request)
+
+
 def launch_daemon():
     """
     Launch the IM daemon
@@ -290,6 +296,7 @@ def launch_daemon():
         server.register_function(ChangeInfrastructureAuth)
         server.register_function(GetInfrastructureOwners)
         server.register_function(EstimateResources)
+        server.register_function(GetStats)
 
         # Launch the API XMLRPC thread
         server.serve_forever_in_thread()