diff --git a/IM/InfrastructureList.py b/IM/InfrastructureList.py
index e24da4ff..21a183c5 100644
--- a/IM/InfrastructureList.py
+++ b/IM/InfrastructureList.py
@@ -250,7 +250,7 @@ def _save_data_to_db(db_url, inf_list, inf_id=None):
return None
@staticmethod
- def _gen_where_from_auth(auth):
+ def _gen_where_from_auth(auth, deleted=0):
like = ""
if auth:
for elem in auth.getAuthInfo('InfrastructureManager'):
@@ -260,12 +260,12 @@ def _gen_where_from_auth(auth):
like += "auth like '%%\"" + elem.get("username") + "\"%%'"
if like:
- return "where deleted = 0 and (" + like + ")"
+ return "where deleted = %d and (%s)" % (deleted, like)
else:
- return "where deleted = 0"
+ return "where deleted = %d" % deleted
@staticmethod
- def _gen_filter_from_auth(auth):
+ def _gen_filter_from_auth(auth, deleted=0):
like = ""
if auth:
for elem in auth.getAuthInfo('InfrastructureManager'):
@@ -275,9 +275,9 @@ def _gen_filter_from_auth(auth):
like += '"%s"' % elem.get("username")
if like:
- return {"deleted": 0, "auth": {"$regex": like}}
+ return {"deleted": deleted, "auth": {"$regex": like}}
else:
- return {"deleted": 0}
+ return {"deleted": deleted}
@staticmethod
def _get_inf_ids_from_db(auth=None):
diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py
index 24a012ac..f0ed96f2 100644
--- a/IM/InfrastructureManager.py
+++ b/IM/InfrastructureManager.py
@@ -42,6 +42,7 @@
from IM.openid.JWT import JWT
from IM.openid.OpenIDClient import OpenIDClient
from IM.vault import VaultCredentials
+from IM.Stats import Stats
if Config.MAX_SIMULTANEOUS_LAUNCHES > 1:
@@ -2036,3 +2037,21 @@ def EstimateResouces(radl_data, auth):
cont += 1
return res
+
+ @staticmethod
+ def GetStats(init_date, end_date, auth):
+ """
+ Get the statistics from the IM DB.
+ Args:
+ - init_date(str): Only will be returned infrastructure created afther this date.
+ - end_date(str): Only will be returned infrastructure created before this date.
+ - auth(Authentication): parsed authentication tokens.
+ Return: a list of dict with the stats.
+ """
+ # First check the auth data
+ auth = InfrastructureManager.check_auth_data(auth)
+ stats = Stats.get_stats(init_date, end_date, auth)
+ if not stats:
+ raise Exception("ERROR connecting with the database!.")
+ else:
+ return stats
diff --git a/IM/REST.py b/IM/REST.py
index 572f3c55..3f92a8a0 100644
--- a/IM/REST.py
+++ b/IM/REST.py
@@ -21,6 +21,7 @@
import flask
import os
import yaml
+import datetime
from cheroot.wsgi import Server as WSGIServer, PathInfoDispatcher
from cheroot.ssl.builtin import BuiltinSSLAdapter
@@ -1077,6 +1078,49 @@ def RESTChangeInfrastructureAuth(infid=None):
return return_error(400, "Error modifying infrastructure owner: %s" % get_ex_error(ex))
+@app.route('/stats', methods=['GET'])
+def RESTGetStats():
+ try:
+ auth = get_auth_header()
+ except Exception:
+ return return_error(401, "No authentication data provided")
+
+ try:
+ init_date = None
+ if "init_date" in flask.request.args.keys():
+ init_date = flask.request.args.get("init_date").lower()
+ init_date = init_date.replace("/", "-")
+ parts = init_date.split("-")
+ try:
+ year = int(parts[0])
+ month = int(parts[1])
+ day = int(parts[2])
+ datetime.date(year, month, day)
+ except Exception:
+ return return_error(400, "Incorrect format in init_date parameter: YYYY/MM/dd")
+ else:
+ init_date = "1970-01-01"
+
+ end_date = None
+ if "end_date" in flask.request.args.keys():
+ end_date = flask.request.args.get("end_date").lower()
+ end_date = end_date.replace("/", "-")
+ parts = end_date.split("-")
+ try:
+ year = int(parts[0])
+ month = int(parts[1])
+ day = int(parts[2])
+ datetime.date(year, month, day)
+ except Exception:
+ return return_error(400, "Incorrect format in end_date parameter: YYYY/MM/dd")
+
+ stats = InfrastructureManager.GetStats(init_date, end_date, auth)
+ return format_output(stats, default_type="application/json", field_name="stats")
+ except Exception as ex:
+ logger.exception("Error getting stats")
+ return return_error(400, "Error getting stats: %s" % get_ex_error(ex))
+
+
@app.errorhandler(403)
def error_mesage_403(error):
return return_error(403, error.description)
diff --git a/IM/ServiceRequests.py b/IM/ServiceRequests.py
index 0f8cfad1..9fba899c 100644
--- a/IM/ServiceRequests.py
+++ b/IM/ServiceRequests.py
@@ -60,6 +60,7 @@ class IMBaseRequest(AsyncRequest):
CHANGE_INFRASTRUCTURE_AUTH = "ChangeInfrastructureAuth"
GET_INFRASTRUCTURE_OWNERS = "GetInfrastructureOwners"
ESTIMATE_RESOURCES = "EstimateResouces"
+ GET_STATS = "GetStats"
@staticmethod
def create_request(function, arguments=()):
@@ -119,6 +120,8 @@ def create_request(function, arguments=()):
return Request_GetInfrastructureOwners(arguments)
elif function == IMBaseRequest.ESTIMATE_RESOURCES:
return Request_EstimateResouces(arguments)
+ elif function == IMBaseRequest.GET_STATS:
+ return Request_GetStats(arguments)
else:
raise NotImplementedError("Function not Implemented")
@@ -473,3 +476,14 @@ def _call_function(self):
self._error_mesage = "Error getting the resources estimation"
(radl_data, auth_data) = self.arguments
return IM.InfrastructureManager.InfrastructureManager.EstimateResouces(radl_data, Authentication(auth_data))
+
+
+class Request_GetStats(IMBaseRequest):
+ """
+ Request class for the GetStats function
+ """
+
+ def _call_function(self):
+ self._error_mesage = "Error getting stats"
+ (init_date, end_date, auth_data) = self.arguments
+ return IM.InfrastructureManager.InfrastructureManager.GetStats(init_date, end_date, Authentication(auth_data))
diff --git a/IM/Stats.py b/IM/Stats.py
new file mode 100644
index 00000000..ff185224
--- /dev/null
+++ b/IM/Stats.py
@@ -0,0 +1,148 @@
+# IM - Infrastructure Manager
+# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import os.path
+import datetime
+import json
+import yaml
+import logging
+
+from IM.db import DataBase
+from IM.auth import Authentication
+from IM.config import Config
+from IM.InfrastructureList import InfrastructureList
+from radl.radl_parse import parse_radl
+
+
+class Stats():
+
+ logger = logging.getLogger('InfrastructureManager')
+ """Logger object."""
+
+ @staticmethod
+ def _get_data(str_data, init_date=None, end_date=None):
+ dic = json.loads(str_data)
+ resp = {'creation_date': None}
+ if 'creation_date' in dic and dic['creation_date']:
+ creation_date = datetime.datetime.fromtimestamp(float(dic['creation_date']))
+ resp['creation_date'] = str(creation_date)
+ if init_date and creation_date < init_date:
+ return None
+ if end_date and creation_date > end_date:
+ return None
+
+ resp['tosca_name'] = None
+ if 'extra_info' in dic and dic['extra_info'] and "TOSCA" in dic['extra_info']:
+ try:
+ tosca = yaml.safe_load(dic['extra_info']['TOSCA'])
+ icon = tosca.get("metadata", {}).get("icon", "")
+ resp['tosca_name'] = os.path.basename(icon)[:-4]
+ except Exception:
+ Stats.logger.exception("Error loading TOSCA.")
+
+ resp['vm_count'] = 0
+ resp['cpu_count'] = 0
+ resp['memory_size'] = 0
+ resp['cloud_type'] = None
+ resp['cloud_host'] = None
+ resp['hybrid'] = False
+ resp['deleted'] = True if 'deleted' in dic and dic['deleted'] else False
+ for str_vm_data in dic['vm_list']:
+ vm_data = json.loads(str_vm_data)
+ cloud_data = json.loads(vm_data["cloud"])
+
+ # only get the cloud of the first VM
+ if not resp['cloud_type']:
+ resp['cloud_type'] = cloud_data["type"]
+ if not resp['cloud_host']:
+ resp['cloud_host'] = cloud_data["server"]
+ elif resp['cloud_host'] != cloud_data["server"]:
+ resp['hybrid'] = True
+
+ vm_sys = parse_radl(vm_data['info']).systems[0]
+ if vm_sys.getValue('cpu.count'):
+ resp['cpu_count'] += vm_sys.getValue('cpu.count')
+ if vm_sys.getValue('memory.size'):
+ resp['memory_size'] += vm_sys.getFeature('memory.size').getValue('M')
+ resp['vm_count'] += 1
+
+ inf_auth = Authentication.deserialize(dic['auth']).getAuthInfo('InfrastructureManager')[0]
+ resp['im_user'] = inf_auth.get('username')
+ return resp
+
+ @staticmethod
+ def get_stats(init_date="1970-01-01", end_date=None, auth=None):
+ """
+ Get the statistics from the IM DB.
+
+ Args:
+
+ - init_date(str): Only will be returned infrastructure created afther this date.
+ - end_date(str): Only will be returned infrastructure created afther this date.
+ - auth(Authentication): parsed authentication tokens.
+
+ Return: a list of dict with the stats with the following format:
+ {'creation_date': '2022-03-07 13:16:14',
+ 'tosca_name': 'kubernetes',
+ 'vm_count': 2,
+ 'cpu_count': 4,
+ 'memory_size': 1024,
+ 'cloud_type': 'OSCAR',
+ 'cloud_host': 'sharp-elbakyan5.im.grycap.net',
+ 'hybrid': False,
+ 'im_user': '__OPENID__mcaballer',
+ 'inf_id': '1',
+ 'deleted': False,
+ 'last_date': '2022-03-23'}
+ """
+ stats = []
+ db = DataBase(Config.DATA_DB)
+ if db.connect():
+ if db.db_type == DataBase.MONGO:
+ filt = InfrastructureList._gen_filter_from_auth(auth, 1)
+ if end_date:
+ filt["date"] = {"$lte": end_date}
+ res = db.find("inf_list", filt, {"id": True, "data": True, "date": True}, [('id', -1)])
+ else:
+ where = InfrastructureList._gen_where_from_auth(auth, 1)
+ if end_date:
+ where += " and date <= '%s'" % end_date
+ res = db.select("select data, date, id from inf_list %s order by rowid desc" % where)
+
+ for elem in res:
+ if db.db_type == DataBase.MONGO:
+ data = elem["data"]
+ date = elem["date"]
+ inf_id = elem["id"]
+ else:
+ data = elem[0]
+ date = elem[1]
+ inf_id = elem[2]
+ try:
+ init = datetime.datetime.strptime(init_date, "%Y-%m-%d")
+ end = datetime.datetime.strptime(end_date, "%Y-%m-%d") if end_date else None
+ res = Stats._get_data(data, init, end)
+ if res:
+ res['inf_id'] = inf_id
+ res['last_date'] = str(date)
+ stats.append(res)
+ except Exception:
+ Stats.logger.exception("ERROR reading infrastructure info from Inf ID: %s" % inf_id)
+ db.close()
+ return stats
+ else:
+ Stats.logger.error("ERROR connecting with the database!.")
+ return None
diff --git a/IM/im_service.py b/IM/im_service.py
index 0edaf452..d7eff4e6 100755
--- a/IM/im_service.py
+++ b/IM/im_service.py
@@ -229,6 +229,12 @@ def EstimateResources(radl_data, auth_data):
return WaitRequest(request)
+def GetStats(init_date, end_date, auth_data):
+ request = IMBaseRequest.create_request(
+ IMBaseRequest.GET_STATS, (init_date, end_date, auth_data))
+ return WaitRequest(request)
+
+
def launch_daemon():
"""
Launch the IM daemon
@@ -290,6 +296,7 @@ def launch_daemon():
server.register_function(ChangeInfrastructureAuth)
server.register_function(GetInfrastructureOwners)
server.register_function(EstimateResources)
+ server.register_function(GetStats)
# Launch the API XMLRPC thread
server.serve_forever_in_thread()
diff --git a/IM/swagger_api.yaml b/IM/swagger_api.yaml
index 114b04d6..168c9c44 100644
--- a/IM/swagger_api.yaml
+++ b/IM/swagger_api.yaml
@@ -2,7 +2,7 @@ openapi: 3.0.0
info:
description: Infrastructure Manager (IM) REST API.
- version: 1.17.0
+ version: 1.18.0
title: Infrastructure Manager (IM) REST API
contact:
email: products@grycap.upv.es
@@ -17,6 +17,8 @@ tags:
description: Get IM server version.
- name: clouds
description: Get cloud information.
+ - name: stats
+ description: Get IM server stats.
paths:
@@ -38,6 +40,58 @@ paths:
'400':
description: Invalid status value
+ /stats:
+ get:
+ tags:
+ - stats
+ summary: Get IM server stats.
+ description: >-
+ Return the stats of the current user in the IM service.
+ Return all the infrastructures in the interval init_date-end_date parameters deployed
+ by the user showing some aggregated information.
+ operationId: GetStats
+ parameters:
+ - name: init_date
+ in: query
+ description: >-
+ The init_date parameter is optional and it is a date with format
+ YYYY/MM/dd. Only will be returned infrastructure created afther this date.
+ required: false
+ schema:
+ type: string
+ - name: end_date
+ in: query
+ description: >-
+ The end_date parameter is optional and it is a date with format
+ YYYY/MM/dd. Only will be returned infrastructure created before this date.
+ required: false
+ schema:
+ type: string
+ responses:
+ '200':
+ description: successful operation
+ content:
+ application/json:
+ examples:
+ response:
+ value:
+ - creation_date: '2022-03-07 13:16:14'
+ tosca_name: 'kubernetes'
+ vm_count: 2
+ cpu_count: 4
+ memory_size: 1024
+ cloud_type: 'OSCAR'
+ cloud_host: 'server.com'
+ hybrid: false
+ deleted: false
+ im_user: 'username'
+ inf_id: '1'
+ last_date: '2022-03-23'
+ '400':
+ description: Invalid status value
+ '401':
+ description: Unauthorized
+
/infrastructures:
get:
tags:
diff --git a/doc/source/REST.rst b/doc/source/REST.rst
index 2bd7b125..4402b48e 100644
--- a/doc/source/REST.rst
+++ b/doc/source/REST.rst
@@ -489,3 +489,31 @@ GET ``http://imserver.com/clouds//quotas``
"security_groups": {"used": 1, "limit": 10}
}
}
+
+GET ``http://imserver.com/stats``
+ :Response Content-type: application/json
+ :ok response: 200 OK
+ :input fields: ``init_date`` (optional)
+ :input fields: ``end_date`` (optional)
+ :fail response: 401, 400
+
+ Return the stats of the current user in the IM service.
+ Return all the infrastructures deployed by the user showing some
+ aggregated information. In JSON format::
+
+ {
+ "stats": [
+ {"creation_date": "2022-03-07 13:16:14",
+ "tosca_name": "kubernetes",
+ "vm_count": 2,
+ "cpu_count": 4,
+ "memory_size": 1024,
+ "cloud_type": "OSCAR",
+ "cloud_host": "server.com",
+ "hybrid": false,
+ "deleted": false,
+ "im_user": "username",
+ "inf_id": "1",
+ "last_date": "2022-03-23"}
+ ]
+ }
diff --git a/doc/source/xmlrpc.rst b/doc/source/xmlrpc.rst
index e993811b..0125fb74 100644
--- a/doc/source/xmlrpc.rst
+++ b/doc/source/xmlrpc.rst
@@ -452,3 +452,29 @@ This is the list of method names:
]
}
}
+
+``GetStat``
+ :parameter 0: ``init_date``: string
+ :parameter 1: ``end_date``: string
+ :parameter 2: ``auth``: array of structs
+ :ok response: [true, list of dicts]
+ :fail response: [false, ``error``: string]
+
+ Return the stats of the current user in the IM service.
+ Return all the infrastructures in the interval ``init_date``-``end-date`` deployed by the user
+ showing some aggregated information. In JSON format::
+
+ [
+ {"creation_date": "2022-03-07 13:16:14",
+ "tosca_name": "kubernetes",
+ "vm_count": 2,
+ "cpu_count": 4,
+ "memory_size": 1024,
+ "cloud_type": "OSCAR",
+ "cloud_host": "server.com",
+ "hybrid": false,
+ "deleted": false,
+ "im_user": "username",
+ "inf_id": "1",
+ "last_date": "2022-03-23"}
+ ]
diff --git a/test/unit/REST.py b/test/unit/REST.py
index 56b24ba4..3b723a9b 100755
--- a/test/unit/REST.py
+++ b/test/unit/REST.py
@@ -838,6 +838,21 @@ def test_GetInfrastructureOwners(self, GetInfrastructureOwners):
res = self.client.get('/infrastructures/1/authorization', headers=headers)
self.assertEqual(res.json, {"authorization": ["user1", "user2"]})
+ @patch("IM.InfrastructureManager.InfrastructureManager.GetStats")
+ def test_GetStats(self, GetStats):
+ """Test REST GetStats."""
+ headers = {"AUTHORIZATION": "type = InfrastructureManager; username = user; password = pass"}
+ GetStats.return_value = [{"key": 1}]
+
+ res = self.client.get('/stats?init_date=2010-01-01&end_date=2022-01-01', headers=headers)
+
+ self.assertEqual(res.json, {"stats": [{"key": 1}]})
+ self.assertEqual(GetStats.call_args_list[0][0][0], '2010-01-01')
+ self.assertEqual(GetStats.call_args_list[0][0][1], '2022-01-01')
+ self.assertEqual(GetStats.call_args_list[0][0][2].auth_list, [{"type": "InfrastructureManager",
+ "username": "user",
+ "password": "pass"}])
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/unit/ServiceRequests.py b/test/unit/ServiceRequests.py
index 4bf16b98..795baf46 100755
--- a/test/unit/ServiceRequests.py
+++ b/test/unit/ServiceRequests.py
@@ -214,6 +214,13 @@ def test_estimate_resources(self, inflist):
IM.ServiceRequests.IMBaseRequest.ESTIMATE_RESOURCES, ("", ""))
req._call_function()
+ @patch('IM.InfrastructureManager.InfrastructureManager')
+ def test_get_stats(self, inflist):
+ import IM.ServiceRequests
+ req = IM.ServiceRequests.IMBaseRequest.create_request(
+ IM.ServiceRequests.IMBaseRequest.GET_STATS, ("", "", ""))
+ req._call_function()
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py
index ffb3383a..17402b4d 100644
--- a/test/unit/test_im_logic.py
+++ b/test/unit/test_im_logic.py
@@ -23,6 +23,7 @@
import sys
import json
import base64
+import yaml
from mock import Mock, patch, MagicMock
@@ -1560,6 +1561,48 @@ def test_estimate_resources(self):
'storage': [{'sizeInGigabytes': 100}]
}})
+ @patch('IM.Stats.DataBase')
+ @patch('IM.InfrastructureManager.InfrastructureManager.check_auth_data')
+ def test_get_stats(self, check_auth_data, DataBase):
+ radl = """
+ system node (
+ memory.size = 512M and
+ cpu.count = 2
+ )"""
+
+ auth = Authentication([{'type': 'InfrastructureManager', 'token': 'atoken',
+ 'username': '__OPENID__mcaballer', 'password': 'pass'}])
+ check_auth_data.return_value = auth
+
+ db = MagicMock()
+ inf_data = {
+ "id": "1",
+ "auth": auth.serialize(),
+ "creation_date": 1646655374,
+ "extra_info": {"TOSCA": yaml.dump({"metadata": {"icon": "kubernetes.png"}})},
+ "vm_list": [
+ json.dumps({"cloud": '{"type": "OSCAR", "server": "sharp-elbakyan5.im.grycap.net"}', "info": radl}),
+ json.dumps({"cloud": '{"type": "OSCAR", "server": "sharp-elbakyan5.im.grycap.net"}', "info": radl})
+ ]
+ }
+ db.select.return_value = [(json.dumps(inf_data), '2022-03-23', '1')]
+ DataBase.return_value = db
+
+ stats = IM.GetStats('2001-01-01', '2122-01-01', auth)
+ expected_res = [{'creation_date': '2022-03-07 12:16:14',
+ 'tosca_name': 'kubernetes',
+ 'vm_count': 2,
+ 'cpu_count': 4,
+ 'memory_size': 1024,
+ 'cloud_type': 'OSCAR',
+ 'cloud_host': 'sharp-elbakyan5.im.grycap.net',
+ 'hybrid': False,
+ 'deleted': False,
+ 'im_user': '__OPENID__mcaballer',
+ 'inf_id': '1',
+ 'last_date': '2022-03-23'}]
+ self.assertEqual(stats, expected_res)
+
if __name__ == "__main__":
unittest.main()