Merge pull request #1504 from grycap/devel

Devel

IM/InfrastructureInfo.py

@@ -184,16 +184,14 @@ def deserialize(str_data):
         return newinf
 
     @staticmethod
-    def deserialize_auth(str_data):
+    def deserialize_auth(inf_id, deleted, str_data):
         """
         Only Loads auth data
         """
         newinf = InfrastructureInfo()
-        dic = json.loads(str_data)
-        newinf.deleted = dic['deleted']
-        newinf.id = dic['id']
-        if dic['auth']:
-            newinf.auth = Authentication.deserialize(dic['auth'])
+        newinf.deleted = deleted
+        newinf.id = inf_id
+        newinf.auth = Authentication.deserialize(str_data)
         return newinf
 
     def destroy_vms(self, auth):

IM/InfrastructureList.py

@@ -63,12 +63,13 @@ def get_inf_ids(auth=None):
         if auth:
             # In this case only loads the auth data to improve performance
             inf_ids = []
-            for inf_id in InfrastructureList._get_inf_ids_from_db():
+            for inf_id in InfrastructureList._get_inf_ids_from_db(auth):
+                inf = None
                 res = InfrastructureList._get_data_from_db(Config.DATA_DB, inf_id, auth)
                 if res:
                     inf = res[inf_id]
-                    if inf and inf.is_authorized(auth):
-                        inf_ids.append(inf.id)
+                if inf and inf.is_authorized(auth):
+                    inf_ids.append(inf.id)
             return inf_ids
         else:
             return InfrastructureList._get_inf_ids_from_db()
@@ -146,10 +147,11 @@ def init_table():
                 InfrastructureList.logger.debug("Creating the IM database!.")
                 if db.db_type == DataBase.MYSQL:
                     db.execute("CREATE TABLE inf_list(rowid INTEGER NOT NULL AUTO_INCREMENT UNIQUE,"
-                               " id VARCHAR(255) PRIMARY KEY, deleted INTEGER, date TIMESTAMP, data LONGBLOB)")
+                               " id VARCHAR(255) PRIMARY KEY, deleted INTEGER, date TIMESTAMP, data LONGBLOB,"
+                               " auth LONGBLOB)")
                 elif db.db_type == DataBase.SQLITE:
                     db.execute("CREATE TABLE inf_list(id VARCHAR(255) PRIMARY KEY, deleted INTEGER,"
-                               " date TIMESTAMP, data LONGBLOB)")
+                               " date TIMESTAMP, data LONGBLOB, auth LONGBLOB)")
                 db.close()
             return True
         else:
@@ -168,25 +170,31 @@ def _get_data_from_db(db_url, inf_id=None, auth=None):
             db = DataBase(db_url)
             if db.connect():
                 inf_list = {}
+                data_field = "data"
+                if auth:
+                    data_field = "auth"
                 if inf_id:
                     if db.db_type == DataBase.MONGO:
-                        res = db.find("inf_list", {"id": inf_id}, {"data": True})
+                        res = db.find("inf_list", {"id": inf_id}, {data_field: True, "deleted": True})
                     else:
-                        res = db.select("select data from inf_list where id = %s", (inf_id,))
+                        res = db.select("select " + data_field + ",deleted from inf_list where id = %s", (inf_id,))
                 else:
                     if db.db_type == DataBase.MONGO:
-                        res = db.find("inf_list", {"deleted": 0}, {"data": True}, [('_id', -1)])
+                        res = db.find("inf_list", {"deleted": 0}, {data_field: True, "deleted": True}, [('_id', -1)])
                     else:
-                        res = db.select("select data from inf_list where deleted = 0 order by rowid desc")
+                        res = db.select("select " + data_field + ",deleted from inf_list where deleted = 0"
+                                        " order by rowid desc")
                 if len(res) > 0:
                     for elem in res:
                         if db.db_type == DataBase.MONGO:
-                            data = elem['data']
+                            data = elem[data_field]
+                            deleted = elem["deleted"]
                         else:
                             data = elem[0]
+                            deleted = elem[1]
                         try:
                             if auth:
-                                inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize_auth(data)
+                                inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize_auth(inf_id, deleted, data)
                             else:
                                 inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize(data)
                             inf_list[inf.id] = inf
@@ -220,10 +228,12 @@ def _save_data_to_db(db_url, inf_list, inf_id=None):
                 data = inf.serialize()
                 if db.db_type == DataBase.MONGO:
                     res = db.replace("inf_list", {"id": inf.id}, {"id": inf.id, "deleted": int(inf.deleted),
-                                                                  "data": data, "date": time.time()})
+                                                                  "data": data, "date": time.time(),
+                                                                  "auth": inf.auth.serialize()})
                 else:
-                    res = db.execute("replace into inf_list (id, deleted, data, date) values (%s, %s, %s, now())",
-                                     (inf.id, int(inf.deleted), data))
+                    res = db.execute("replace into inf_list (id, deleted, data, date, auth)"
+                                     " values (%s, %s, %s, now(), %s)",
+                                     (inf.id, int(inf.deleted), data, inf.auth.serialize()))
 
             db.close()
             return res
@@ -232,15 +242,47 @@ def _save_data_to_db(db_url, inf_list, inf_id=None):
             return None
 
     @staticmethod
-    def _get_inf_ids_from_db():
+    def _gen_where_from_auth(auth):
+        like = ""
+        if auth:
+            for elem in auth.getAuthInfo('InfrastructureManager'):
+                if elem.get("username"):
+                    if like:
+                        like += " or "
+                    like += "auth like '%%" + elem.get("username") + "%%'"
+
+        if like:
+            return "where deleted = 0 and (" + like + ")"
+        else:
+            return "where deleted = 0"
+
+    @staticmethod
+    def _gen_filter_from_auth(auth):
+        like = ""
+        if auth:
+            for elem in auth.getAuthInfo('InfrastructureManager'):
+                if elem.get("username"):
+                    if like:
+                        like += "|"
+                    like += elem.get("username")
+
+        if like:
+            return {"deleted": 0, "auth": {"$regex": like}}
+        else:
+            return {"deleted": 0}
+
+    @staticmethod
+    def _get_inf_ids_from_db(auth=None):
         try:
             db = DataBase(Config.DATA_DB)
             if db.connect():
                 inf_list = []
                 if db.db_type == DataBase.MONGO:
-                    res = db.find("inf_list", {"deleted": 0}, {"id": True}, [('id', -1)])
+                    filt = InfrastructureList._gen_filter_from_auth(auth)
+                    res = db.find("inf_list", filt, {"id": True}, [('id', -1)])
                 else:
-                    res = db.select("select id from inf_list where deleted = 0 order by rowid desc")
+                    where = InfrastructureList._gen_where_from_auth(auth)
+                    res = db.select("select id from inf_list %s order by rowid desc" % where)
                 for elem in res:
                     if db.db_type == DataBase.MONGO:
                         inf_list.append(elem['id'])

IM/__init__.py

@@ -19,7 +19,7 @@
            'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry',
            'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPAdapter',
            'VirtualMachine', 'VMRC', 'xmlobject']
-__version__ = '1.14.1'
+__version__ = '1.15.0'
 __author__ = 'Miguel Caballer'
 
 

IM/connectors/OSCAR.py

@@ -138,6 +138,8 @@ def _get_service_json(radl_system):
             service["cpu"] = "%g" % radl_system.getValue("cpu.count")
         if radl_system.getValue("gpu.count"):
             service["enable_gpu"] = True
+        if radl_system.getValue("cpu.sgx"):
+            service["enable_sgx"] = True
         if radl_system.getValue("script"):
             service["script"] = radl_system.getValue("script")
         if radl_system.getValue("alpine"):
@@ -148,6 +150,13 @@ def _get_service_json(radl_system):
                 secrets = [secrets]
             service["image_pull_secrets"] = secrets
 
+        expose = {}
+        for elem in ["min_scale", "max_scale", "port", "cpu_threshold"]:
+            if radl_system.getValue("expose.%s" % elem):
+                expose[elem] = radl_system.getValue("expose.%s" % elem)
+        if expose:
+            service["expose"] = expose
+
         if radl_system.getValue("disk.0.image.url"):
             url_image = urlparse(radl_system.getValue("disk.0.image.url"))
             image = ""

IM/db.py

@@ -281,6 +281,17 @@ def replace(self, table_name, filt, replacement):
             res = self.connection[table_name].replace_one(filt, replacement, True)
             return res.modified_count == 1 or res.upserted_id is not None
 
+    def update(self, table_name, filt, updates):
+        """ insert/replace elements """
+        if self.db_type != DataBase.MONGO:
+            raise Exception("Operation only supported in MongoDB")
+
+        if self.connection is None:
+            raise Exception("DataBase object not connected")
+        else:
+            res = self.connection[table_name].update_one(filt, updates, True)
+            return res.modified_count == 1 or res.upserted_id is not None
+
     def delete(self, table_name, filt):
         """ delete elements """
         if self.db_type != DataBase.MONGO:

IM/tosca/Tosca.py

@@ -3,7 +3,7 @@
 import yaml
 import copy
 import operator
-import requests
+import requests_cache
 import json
 import re
 from toscaparser.nodetemplate import NodeTemplate
@@ -36,10 +36,12 @@ class Tosca:
 
     ARTIFACTS_PATH = os.path.dirname(os.path.realpath(__file__)) + "/tosca-types/artifacts"
     ARTIFACTS_REMOTE_REPO = "https://raw.githubusercontent.com/indigo-dc/tosca-types/master/artifacts/"
+    GET_TIMEOUT = 20
 
     logger = logging.getLogger('InfrastructureManager')
 
     def __init__(self, yaml_str, verify=True):
+        self.cache_session = requests_cache.CachedSession('tosca_cache', cache_control=True, expire_after=3600)
         Tosca.logger.debug("TOSCA: %s" % yaml_str)
         self.yaml = yaml.safe_load(yaml_str)
         if not verify:
@@ -796,7 +798,7 @@ def _gen_configure_from_interfaces(self, node, compute, interfaces):
                 if implementation_url[0] in ['http', 'https', 'ftp']:
                     script_path = implementation_url[2]
                     try:
-                        resp = requests.get(implementation)
+                        resp = self.cache_session.get(implementation, timeout=self.GET_TIMEOUT)
                         script_content = resp.text
                         if resp.status_code != 200:
                             raise Exception(resp.reason + "\n" + resp.text)
@@ -814,7 +816,8 @@ def _gen_configure_from_interfaces(self, node, compute, interfaces):
                         f.close()
                     else:
                         try:
-                            resp = requests.get(Tosca.ARTIFACTS_REMOTE_REPO + implementation)
+                            resp = self.cache_session.get(Tosca.ARTIFACTS_REMOTE_REPO + implementation,
+                                                          timeout=self.GET_TIMEOUT)
                             script_content = resp.text
                             if resp.status_code != 200:
                                 raise Exception(resp.reason + "\n" + resp.text)
@@ -1990,6 +1993,9 @@ def _gen_oscar_system(self, node):
                 elif prop.name == 'enable_gpu':
                     if value:
                         res.setValue('gpu.count', 1)
+                elif prop.name == 'enable_sgx':
+                    if value:
+                        res.setValue('cpu.sgx', 1)
                 elif prop.name == 'memory':
                     if not value.endswith("B"):
                         value += "B"
@@ -2020,6 +2026,10 @@ def _gen_oscar_system(self, node):
                                              'verify', 'oneprovider_host', 'token', 'space']:
                                     if provider.get(elem):
                                         res.setValue("%s.%s" % (provider_pref, elem), provider.get(elem))
+                elif prop.name == 'exposed':
+                    for elem in ["min_scale", "max_scale", "port", "cpu_threshold"]:
+                        if elem in value:
+                            res.setValue("expose.%s" % elem, value)
                 else:
                     # this should never happen
                     Tosca.logger.warn("Property %s not expected. Ignoring." % prop.name)

changelog

@@ -738,8 +738,13 @@ IM 1.14.0:
     * Enable to install ansible collections.
     * Enable to use wildcards in the instance type
 
-IM 1.14.1:
+IM 1.15.0:
     * Fix error resizing VMs.
     * Enable to set root disk size in EC2, Azure and GCE conns.
     * Return error if instance is not get in stop, start, reboot ops in EC2 conn
-    * Fix error in OSCAR or Lambda cons with env variables with ":".
+    * Fix error in OSCAR or Lambda cons with env variables with ":"
+    * Improve performance in Infrastructure List operation.
+        WARNING: It requires a DB update.
+        Please make a copy of the DB before applying the script.
+        Use scripts/db_1_14_X_to_1_15_X.py to update it.
+    * Add requests-cache to improve performance in TOSCA parsing

codemeta.json

@@ -6,7 +6,7 @@
     "@type": "SoftwareSourceCode",
     "identifier": "im",
     "name": "Infrastructure Manager",
-    "version": "1.14.1",
+    "version": "1.15.0",
     "description": "IM is a tool that deploys complex and customized virtual infrastructures on IaaS Cloud deployments",
     "license": "GNU General Public License v3.0",
     "author": [

doc/source/intro.rst

@@ -33,6 +33,7 @@ from the `Universitat Politècnica de València (UPV) <http://www.upv.es>`_.
 The following publications summarise both the development and integration in larger architecture. Please acknowledge the usage of this software by citing the first reference:
 
 * Caballer, M.; Blanquer, I.; Moltó, G.; and de Alfonso, C.; **"Dynamic management of virtual infrastructures"** . Journal of Grid Computing, Volume 13, Issue 1, Pages 53-70, 2015, ISSN 1570-7873, 10.1007/s10723-014-9296-5.
+* Caballer, M.; Moltó, G.; Calatrava, A.; and de Blanquer, I.; **"Infrastructure Manager: A TOSCA-Based Orchestrator for the Computing Continuum"** . Journal of Grid Computing, Volume 21, Issue 3, 2023, ISSN 1570-7873, 10.1007/s10723-023-09686-7.
 * Caballer, M.; de Alfonso, C.; Moltó, G.; Romero, E.; Blanquer, I.; and García, A.; **"CodeCloud: A Platform to Enable Execution of Programming Models on the Clouds"** . Journal of Systems and Software, Volume 93, Pages 187-198, 2014, ISSN 0164-1212, 10.1016/j.jss.2014.02.005
 * Caballer, M., Segrelles, D., Moltó, G. and Blanquer, I.  **"A Platform to Deploy Customized Scientific Virtual Infrastructures on the Cloud"** . Concurrency and Computation: Practice and Experience, Volume 27, Issue 16, Pages 4318-4329, 2015, ISSN 1532-0626, 10.1002/cpe.3518.
 * Caballer, M.; de Alfonso, C.; Alvarruiz, F. and Moltó, G.; **"EC3: Elastic Cloud Computing Cluster"** . Journal of Computer and System Sciences, Volume 78, Issue 8, December 2013, Pages 1341-1351, ISSN 0022-0000, 10.1016/j.jcss.2013.06.005.

doc/source/tosca.rst

@@ -41,7 +41,7 @@ using an Ubuntu 20.04 image. As outputs the TOSCA files will return the public I
               properties:
                 type: linux
                 distribution: ubuntu
-                version: 20.04
+                version: 22.04
 
       outputs:
         node_ip:
@@ -276,4 +276,25 @@ and, optionally, an availability zone::
         properties: { cloud_id: cloudid2, availability_zone: some_zone }
         targets: [ compute_three ]
 
-    ...
+    ...
+
+Advanced Output values
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``tosca.nodes.indigo.Compute`` node type adds a new
+attribute named: ``ansible_output``. It is a map that has one element per each IM
+configuration step, so you can access it by name. The steps have the keyword
+``tasks`` that is also a map that has one element per ansible task. In this case
+it can bes accessed using the task name as defined in the playbook. Finally
+there is an ``output`` keyword that returns the output of the task.
+In most of the cases the task is a ``debug`` ansible task that shows anything you
+want to return.
+
+In the following example the specified task was a debug ansible task that shows the
+value of a internal defined value::
+
+    ...
+
+      outputs:
+        node_ip:
+          value: { get_attribute: [ front, ansible_output, lrms_front_end_front_conf_front, tasks, 'grycap.nomad : nomad_secret_id', output ] }

docker-devel/Dockerfile

@@ -2,7 +2,7 @@
 FROM ubuntu:22.04
 ARG BRANCH=devel
 LABEL maintainer="Miguel Caballer <[email protected]>"
-LABEL version="1.14.1"
+LABEL version="1.15.0"
 LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)"
 EXPOSE 8899 8800
 

docker-py3/Dockerfile

@@ -1,7 +1,7 @@
 # Dockerfile to create a container with the IM service
 FROM ubuntu:22.04
 LABEL maintainer="Miguel Caballer <[email protected]>"
-LABEL version="1.14.1"
+LABEL version="1.15.0"
 LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)"
 EXPOSE 8899 8800
 
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3 python3
 RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \
      pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \
      pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.4.0&& \
-     pip3 install IM==1.14.1 && \
+     pip3 install IM==1.15.0 && \
      apt-get purge -y python3-pip git && \
      apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/
 

docker-py3/Dockerfile.alp

@@ -1,7 +1,7 @@
 # Dockerfile to create a container with the IM service
 FROM alpine:3.16
 LABEL maintainer="Miguel Caballer <[email protected]>"
-LABEL version="1.14.1"
+LABEL version="1.15.0"
 LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)"
 EXPOSE 8899 8800
 

requirements-tests.txt

@@ -33,4 +33,5 @@ psutil
 scar
 nose
 mock
-coverage
+coverage
+requests-cache