diff --git a/IM/InfrastructureInfo.py b/IM/InfrastructureInfo.py index 9e26b9c5c..805cd8d8f 100644 --- a/IM/InfrastructureInfo.py +++ b/IM/InfrastructureInfo.py @@ -184,16 +184,14 @@ def deserialize(str_data): return newinf @staticmethod - def deserialize_auth(str_data): + def deserialize_auth(inf_id, deleted, str_data): """ Only Loads auth data """ newinf = InfrastructureInfo() - dic = json.loads(str_data) - newinf.deleted = dic['deleted'] - newinf.id = dic['id'] - if dic['auth']: - newinf.auth = Authentication.deserialize(dic['auth']) + newinf.deleted = deleted + newinf.id = inf_id + newinf.auth = Authentication.deserialize(str_data) return newinf def destroy_vms(self, auth): diff --git a/IM/InfrastructureList.py b/IM/InfrastructureList.py index 86d3d1c9f..70c2a2045 100644 --- a/IM/InfrastructureList.py +++ b/IM/InfrastructureList.py @@ -63,12 +63,13 @@ def get_inf_ids(auth=None): if auth: # In this case only loads the auth data to improve performance inf_ids = [] - for inf_id in InfrastructureList._get_inf_ids_from_db(): + for inf_id in InfrastructureList._get_inf_ids_from_db(auth): + inf = None res = InfrastructureList._get_data_from_db(Config.DATA_DB, inf_id, auth) if res: inf = res[inf_id] - if inf and inf.is_authorized(auth): - inf_ids.append(inf.id) + if inf and inf.is_authorized(auth): + inf_ids.append(inf.id) return inf_ids else: return InfrastructureList._get_inf_ids_from_db() @@ -146,10 +147,11 @@ def init_table(): InfrastructureList.logger.debug("Creating the IM database!.") if db.db_type == DataBase.MYSQL: db.execute("CREATE TABLE inf_list(rowid INTEGER NOT NULL AUTO_INCREMENT UNIQUE," - " id VARCHAR(255) PRIMARY KEY, deleted INTEGER, date TIMESTAMP, data LONGBLOB)") + " id VARCHAR(255) PRIMARY KEY, deleted INTEGER, date TIMESTAMP, data LONGBLOB," + " auth LONGBLOB)") elif db.db_type == DataBase.SQLITE: db.execute("CREATE TABLE inf_list(id VARCHAR(255) PRIMARY KEY, deleted INTEGER," - " date TIMESTAMP, data LONGBLOB)") + " date TIMESTAMP, data LONGBLOB, auth LONGBLOB)") db.close() return True else: @@ -168,25 +170,31 @@ def _get_data_from_db(db_url, inf_id=None, auth=None): db = DataBase(db_url) if db.connect(): inf_list = {} + data_field = "data" + if auth: + data_field = "auth" if inf_id: if db.db_type == DataBase.MONGO: - res = db.find("inf_list", {"id": inf_id}, {"data": True}) + res = db.find("inf_list", {"id": inf_id}, {data_field: True, "deleted": True}) else: - res = db.select("select data from inf_list where id = %s", (inf_id,)) + res = db.select("select " + data_field + ",deleted from inf_list where id = %s", (inf_id,)) else: if db.db_type == DataBase.MONGO: - res = db.find("inf_list", {"deleted": 0}, {"data": True}, [('_id', -1)]) + res = db.find("inf_list", {"deleted": 0}, {data_field: True, "deleted": True}, [('_id', -1)]) else: - res = db.select("select data from inf_list where deleted = 0 order by rowid desc") + res = db.select("select " + data_field + ",deleted from inf_list where deleted = 0" + " order by rowid desc") if len(res) > 0: for elem in res: if db.db_type == DataBase.MONGO: - data = elem['data'] + data = elem[data_field] + deleted = elem["deleted"] else: data = elem[0] + deleted = elem[1] try: if auth: - inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize_auth(data) + inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize_auth(inf_id, deleted, data) else: inf = IM.InfrastructureInfo.InfrastructureInfo.deserialize(data) inf_list[inf.id] = inf @@ -220,10 +228,12 @@ def _save_data_to_db(db_url, inf_list, inf_id=None): data = inf.serialize() if db.db_type == DataBase.MONGO: res = db.replace("inf_list", {"id": inf.id}, {"id": inf.id, "deleted": int(inf.deleted), - "data": data, "date": time.time()}) + "data": data, "date": time.time(), + "auth": inf.auth.serialize()}) else: - res = db.execute("replace into inf_list (id, deleted, data, date) values (%s, %s, %s, now())", - (inf.id, int(inf.deleted), data)) + res = db.execute("replace into inf_list (id, deleted, data, date, auth)" + " values (%s, %s, %s, now(), %s)", + (inf.id, int(inf.deleted), data, inf.auth.serialize())) db.close() return res @@ -232,15 +242,47 @@ def _save_data_to_db(db_url, inf_list, inf_id=None): return None @staticmethod - def _get_inf_ids_from_db(): + def _gen_where_from_auth(auth): + like = "" + if auth: + for elem in auth.getAuthInfo('InfrastructureManager'): + if elem.get("username"): + if like: + like += " or " + like += "auth like '%%" + elem.get("username") + "%%'" + + if like: + return "where deleted = 0 and (" + like + ")" + else: + return "where deleted = 0" + + @staticmethod + def _gen_filter_from_auth(auth): + like = "" + if auth: + for elem in auth.getAuthInfo('InfrastructureManager'): + if elem.get("username"): + if like: + like += "|" + like += elem.get("username") + + if like: + return {"deleted": 0, "auth": {"$regex": like}} + else: + return {"deleted": 0} + + @staticmethod + def _get_inf_ids_from_db(auth=None): try: db = DataBase(Config.DATA_DB) if db.connect(): inf_list = [] if db.db_type == DataBase.MONGO: - res = db.find("inf_list", {"deleted": 0}, {"id": True}, [('id', -1)]) + filt = InfrastructureList._gen_filter_from_auth(auth) + res = db.find("inf_list", filt, {"id": True}, [('id', -1)]) else: - res = db.select("select id from inf_list where deleted = 0 order by rowid desc") + where = InfrastructureList._gen_where_from_auth(auth) + res = db.select("select id from inf_list %s order by rowid desc" % where) for elem in res: if db.db_type == DataBase.MONGO: inf_list.append(elem['id']) diff --git a/IM/__init__.py b/IM/__init__.py index c4b1e1320..2dcd0548d 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -19,7 +19,7 @@ 'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry', 'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPAdapter', 'VirtualMachine', 'VMRC', 'xmlobject'] -__version__ = '1.14.1' +__version__ = '1.15.0' __author__ = 'Miguel Caballer' diff --git a/IM/connectors/OSCAR.py b/IM/connectors/OSCAR.py index 2123b6a3a..cd93529ae 100644 --- a/IM/connectors/OSCAR.py +++ b/IM/connectors/OSCAR.py @@ -138,6 +138,8 @@ def _get_service_json(radl_system): service["cpu"] = "%g" % radl_system.getValue("cpu.count") if radl_system.getValue("gpu.count"): service["enable_gpu"] = True + if radl_system.getValue("cpu.sgx"): + service["enable_sgx"] = True if radl_system.getValue("script"): service["script"] = radl_system.getValue("script") if radl_system.getValue("alpine"): @@ -148,6 +150,13 @@ def _get_service_json(radl_system): secrets = [secrets] service["image_pull_secrets"] = secrets + expose = {} + for elem in ["min_scale", "max_scale", "port", "cpu_threshold"]: + if radl_system.getValue("expose.%s" % elem): + expose[elem] = radl_system.getValue("expose.%s" % elem) + if expose: + service["expose"] = expose + if radl_system.getValue("disk.0.image.url"): url_image = urlparse(radl_system.getValue("disk.0.image.url")) image = "" diff --git a/IM/db.py b/IM/db.py index 8a27e3885..1685330a7 100644 --- a/IM/db.py +++ b/IM/db.py @@ -281,6 +281,17 @@ def replace(self, table_name, filt, replacement): res = self.connection[table_name].replace_one(filt, replacement, True) return res.modified_count == 1 or res.upserted_id is not None + def update(self, table_name, filt, updates): + """ insert/replace elements """ + if self.db_type != DataBase.MONGO: + raise Exception("Operation only supported in MongoDB") + + if self.connection is None: + raise Exception("DataBase object not connected") + else: + res = self.connection[table_name].update_one(filt, updates, True) + return res.modified_count == 1 or res.upserted_id is not None + def delete(self, table_name, filt): """ delete elements """ if self.db_type != DataBase.MONGO: diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 21d79f0c6..2e564aea9 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -3,7 +3,7 @@ import yaml import copy import operator -import requests +import requests_cache import json import re from toscaparser.nodetemplate import NodeTemplate @@ -36,10 +36,12 @@ class Tosca: ARTIFACTS_PATH = os.path.dirname(os.path.realpath(__file__)) + "/tosca-types/artifacts" ARTIFACTS_REMOTE_REPO = "https://raw.githubusercontent.com/indigo-dc/tosca-types/master/artifacts/" + GET_TIMEOUT = 20 logger = logging.getLogger('InfrastructureManager') def __init__(self, yaml_str, verify=True): + self.cache_session = requests_cache.CachedSession('tosca_cache', cache_control=True, expire_after=3600) Tosca.logger.debug("TOSCA: %s" % yaml_str) self.yaml = yaml.safe_load(yaml_str) if not verify: @@ -796,7 +798,7 @@ def _gen_configure_from_interfaces(self, node, compute, interfaces): if implementation_url[0] in ['http', 'https', 'ftp']: script_path = implementation_url[2] try: - resp = requests.get(implementation) + resp = self.cache_session.get(implementation, timeout=self.GET_TIMEOUT) script_content = resp.text if resp.status_code != 200: raise Exception(resp.reason + "\n" + resp.text) @@ -814,7 +816,8 @@ def _gen_configure_from_interfaces(self, node, compute, interfaces): f.close() else: try: - resp = requests.get(Tosca.ARTIFACTS_REMOTE_REPO + implementation) + resp = self.cache_session.get(Tosca.ARTIFACTS_REMOTE_REPO + implementation, + timeout=self.GET_TIMEOUT) script_content = resp.text if resp.status_code != 200: raise Exception(resp.reason + "\n" + resp.text) @@ -1990,6 +1993,9 @@ def _gen_oscar_system(self, node): elif prop.name == 'enable_gpu': if value: res.setValue('gpu.count', 1) + elif prop.name == 'enable_sgx': + if value: + res.setValue('cpu.sgx', 1) elif prop.name == 'memory': if not value.endswith("B"): value += "B" @@ -2020,6 +2026,10 @@ def _gen_oscar_system(self, node): 'verify', 'oneprovider_host', 'token', 'space']: if provider.get(elem): res.setValue("%s.%s" % (provider_pref, elem), provider.get(elem)) + elif prop.name == 'exposed': + for elem in ["min_scale", "max_scale", "port", "cpu_threshold"]: + if elem in value: + res.setValue("expose.%s" % elem, value) else: # this should never happen Tosca.logger.warn("Property %s not expected. Ignoring." % prop.name) diff --git a/changelog b/changelog index 64ecfe653..a440652df 100644 --- a/changelog +++ b/changelog @@ -738,8 +738,13 @@ IM 1.14.0: * Enable to install ansible collections. * Enable to use wildcards in the instance type -IM 1.14.1: +IM 1.15.0: * Fix error resizing VMs. * Enable to set root disk size in EC2, Azure and GCE conns. * Return error if instance is not get in stop, start, reboot ops in EC2 conn - * Fix error in OSCAR or Lambda cons with env variables with ":". \ No newline at end of file + * Fix error in OSCAR or Lambda cons with env variables with ":" + * Improve performance in Infrastructure List operation. + WARNING: It requires a DB update. + Please make a copy of the DB before applying the script. + Use scripts/db_1_14_X_to_1_15_X.py to update it. + * Add requests-cache to improve performance in TOSCA parsing \ No newline at end of file diff --git a/codemeta.json b/codemeta.json index 51d7c8aa2..4d8a39d88 100644 --- a/codemeta.json +++ b/codemeta.json @@ -6,7 +6,7 @@ "@type": "SoftwareSourceCode", "identifier": "im", "name": "Infrastructure Manager", - "version": "1.14.1", + "version": "1.15.0", "description": "IM is a tool that deploys complex and customized virtual infrastructures on IaaS Cloud deployments", "license": "GNU General Public License v3.0", "author": [ diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4afafa8ed..33dc29c11 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -33,6 +33,7 @@ from the `Universitat Politècnica de València (UPV) `_. The following publications summarise both the development and integration in larger architecture. Please acknowledge the usage of this software by citing the first reference: * Caballer, M.; Blanquer, I.; Moltó, G.; and de Alfonso, C.; **"Dynamic management of virtual infrastructures"** . Journal of Grid Computing, Volume 13, Issue 1, Pages 53-70, 2015, ISSN 1570-7873, 10.1007/s10723-014-9296-5. +* Caballer, M.; Moltó, G.; Calatrava, A.; and de Blanquer, I.; **"Infrastructure Manager: A TOSCA-Based Orchestrator for the Computing Continuum"** . Journal of Grid Computing, Volume 21, Issue 3, 2023, ISSN 1570-7873, 10.1007/s10723-023-09686-7. * Caballer, M.; de Alfonso, C.; Moltó, G.; Romero, E.; Blanquer, I.; and García, A.; **"CodeCloud: A Platform to Enable Execution of Programming Models on the Clouds"** . Journal of Systems and Software, Volume 93, Pages 187-198, 2014, ISSN 0164-1212, 10.1016/j.jss.2014.02.005 * Caballer, M., Segrelles, D., Moltó, G. and Blanquer, I. **"A Platform to Deploy Customized Scientific Virtual Infrastructures on the Cloud"** . Concurrency and Computation: Practice and Experience, Volume 27, Issue 16, Pages 4318-4329, 2015, ISSN 1532-0626, 10.1002/cpe.3518. * Caballer, M.; de Alfonso, C.; Alvarruiz, F. and Moltó, G.; **"EC3: Elastic Cloud Computing Cluster"** . Journal of Computer and System Sciences, Volume 78, Issue 8, December 2013, Pages 1341-1351, ISSN 0022-0000, 10.1016/j.jcss.2013.06.005. diff --git a/doc/source/tosca.rst b/doc/source/tosca.rst index dde762dd3..049b765df 100644 --- a/doc/source/tosca.rst +++ b/doc/source/tosca.rst @@ -41,7 +41,7 @@ using an Ubuntu 20.04 image. As outputs the TOSCA files will return the public I properties: type: linux distribution: ubuntu - version: 20.04 + version: 22.04 outputs: node_ip: @@ -276,4 +276,25 @@ and, optionally, an availability zone:: properties: { cloud_id: cloudid2, availability_zone: some_zone } targets: [ compute_three ] - ... \ No newline at end of file + ... + +Advanced Output values +^^^^^^^^^^^^^^^^^^^^^^^ + +The ``tosca.nodes.indigo.Compute`` node type adds a new +attribute named: ``ansible_output``. It is a map that has one element per each IM +configuration step, so you can access it by name. The steps have the keyword +``tasks`` that is also a map that has one element per ansible task. In this case +it can bes accessed using the task name as defined in the playbook. Finally +there is an ``output`` keyword that returns the output of the task. +In most of the cases the task is a ``debug`` ansible task that shows anything you +want to return. + +In the following example the specified task was a debug ansible task that shows the +value of a internal defined value:: + + ... + + outputs: + node_ip: + value: { get_attribute: [ front, ansible_output, lrms_front_end_front_conf_front, tasks, 'grycap.nomad : nomad_secret_id', output ] } diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index ce5ee49f4..3b35365d5 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:22.04 ARG BRANCH=devel LABEL maintainer="Miguel Caballer " -LABEL version="1.14.1" +LABEL version="1.15.0" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 103be7440..8f17f1390 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM ubuntu:22.04 LABEL maintainer="Miguel Caballer " -LABEL version="1.14.1" +LABEL version="1.15.0" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3 python3 RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.4.0&& \ - pip3 install IM==1.14.1 && \ + pip3 install IM==1.15.0 && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ diff --git a/docker-py3/Dockerfile.alp b/docker-py3/Dockerfile.alp index f439d43ff..63775f0cc 100644 --- a/docker-py3/Dockerfile.alp +++ b/docker-py3/Dockerfile.alp @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM alpine:3.16 LABEL maintainer="Miguel Caballer " -LABEL version="1.14.1" +LABEL version="1.15.0" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 diff --git a/requirements-tests.txt b/requirements-tests.txt index bffc28f8c..60573b552 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -33,4 +33,5 @@ psutil scar nose mock -coverage \ No newline at end of file +coverage +requests-cache \ No newline at end of file diff --git a/scripts/db_1_14_X_to_1_15_X.py b/scripts/db_1_14_X_to_1_15_X.py new file mode 100644 index 000000000..2282d8c63 --- /dev/null +++ b/scripts/db_1_14_X_to_1_15_X.py @@ -0,0 +1,95 @@ +# IM - Infrastructure Manager +# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import json + +sys.path.append("..") +sys.path.append(".") + +from IM.InfrastructureInfo import InfrastructureInfo +from IM.db import DataBase +from IM.auth import Authentication + + +def get_data_from_db(db, inf_id): + """ + Get data from DB. + """ + + if db.db_type == DataBase.MONGO: + res = db.find("inf_list", {"id": inf_id}, {"data": True}) + else: + res = db.select("select data from inf_list where id = %s", (inf_id,)) + + if len(res) > 0: + elem = res[0] + if db.db_type == DataBase.MONGO: + data = elem["data"] + else: + data = elem[0] + try: + newinf = InfrastructureInfo() + dic = json.loads(data) + newinf.deleted = dic['deleted'] + newinf.id = dic['id'] + if dic['auth']: + newinf.auth = Authentication.deserialize(dic['auth']) + return newinf + except Exception: + print("ERROR reading infrastructure from database, ignoring it!.") + else: + return None + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.stderr.write("Usage: %s \n" % sys.argv[0]) + sys.exit(-1) + + DATA_DB = sys.argv[1] + + db = DataBase(DATA_DB) + if db.connect(): + if db.table_exists("inf_list"): + if db.db_type != DataBase.MONGO: + sys.stdout.write("Updating DB: %s.\n" % DATA_DB) + db.execute("ALTER TABLE `inf_list` ADD COLUMN `auth` BLOB;") + + infs = [] + res = db.select("select id from inf_list order by rowid desc") + for elem in res: + inf_id = elem[0] + try: + inf = get_data_from_db(db, inf_id) + print(inf_id) + if inf: + auth = inf.auth.serialize() + if db.db_type == DataBase.MONGO: + res = db.update("inf_list", {"id": inf_id}, {"auth": auth}) + else: + res = db.execute("UPDATE `inf_list` SET `auth` = %s WHERE id = %s", (auth, inf_id)) + except Exception: + sys.stderr.write("Error updating auth field in Inf ID: %s. Ignoring.\n" % inf_id) + else: + sys.stdout.write("There are no inf_list table. Do not need to update.") + + db.close() + else: + sys.stderr.write("Error connecting with DB: %s\n" % DATA_DB) + sys.exit(-1) + + sys.exit(0) diff --git a/setup.py b/setup.py index 86d4cb48e..5a4e24ffa 100644 --- a/setup.py +++ b/setup.py @@ -70,5 +70,5 @@ install_requires=["ansible >=2.4", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot", "boto >= 2.29", "apache-libcloud >= 3.2.0", "RADL >= 1.1.5", "bottle", "netaddr", "requests >= 2.19", "scp", "tosca-parser", 'defusedxml', 'urllib3>=1.23', 'hvac', - 'psutil', 'scar'] + 'psutil', 'scar', 'requests-cache >= 1.0.0'] ) diff --git a/test/files/tosca_add.yml b/test/files/tosca_add.yml index 81e87fc7c..5e7231ea4 100644 --- a/test/files/tosca_add.yml +++ b/test/files/tosca_add.yml @@ -58,7 +58,7 @@ topology_template: # host Operating System image properties type: linux distribution: ubuntu - version: 16.04 + version: 22.04 test_db: type: tosca.nodes.indigo.Database.MySQL @@ -107,8 +107,8 @@ topology_template: architecture: x86_64 type: linux distribution: ubuntu - version: 18.04 + version: 22.04 outputs: server_url: - value: { get_attribute: [ web_server, public_address ] } \ No newline at end of file + value: { get_attribute: [ web_server, public_address ] } diff --git a/test/files/tosca_ansible_host.yaml b/test/files/tosca_ansible_host.yaml index b21ebb95e..b471cd5eb 100644 --- a/test/files/tosca_ansible_host.yaml +++ b/test/files/tosca_ansible_host.yaml @@ -1,7 +1,7 @@ tosca_definitions_version: tosca_simple_yaml_1_0 imports: - - ec3_custom_types: https://raw.githubusercontent.com/grycap/ec3/tosca/tosca/custom_types.yaml + - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/main/custom_types.yaml topology_template: diff --git a/test/files/tosca_create.yml b/test/files/tosca_create.yml index c79922010..1c72aee4f 100644 --- a/test/files/tosca_create.yml +++ b/test/files/tosca_create.yml @@ -58,7 +58,7 @@ topology_template: # host Operating System image properties type: linux distribution: ubuntu - version: 18.04 + version: 22.04 test_db: type: tosca.nodes.indigo.Database.MySQL @@ -103,7 +103,7 @@ topology_template: properties: type: linux distribution: ubuntu - version: 18.04 + version: 22.04 outputs: server_url: @@ -113,4 +113,4 @@ topology_template: server_creds_password: value: { get_attribute: [ web_server, endpoint, credential, 0, token ] } ansible_output: - value: { get_attribute: [ web_server, ansible_output, main_front, tasks, debug, output ] } \ No newline at end of file + value: { get_attribute: [ web_server, ansible_output, main_front, tasks, debug, output ] } diff --git a/test/files/tosca_oscar.yml b/test/files/tosca_oscar.yml index 0ce207e82..c38b60e36 100644 --- a/test/files/tosca_oscar.yml +++ b/test/files/tosca_oscar.yml @@ -19,6 +19,7 @@ topology_template: echo "Hola" cpu: 0.5 enable_gpu: true + enable_sgx: true image: grycap/image input: - storage_provider: minio.default diff --git a/test/files/tosca_remove.yml b/test/files/tosca_remove.yml index 8b68b95f8..ce933966d 100644 --- a/test/files/tosca_remove.yml +++ b/test/files/tosca_remove.yml @@ -59,7 +59,7 @@ topology_template: # host Operating System image properties type: linux distribution: ubuntu - version: 16.04 + version: 22.04 test_db: type: tosca.nodes.indigo.Database.MySQL @@ -110,8 +110,8 @@ topology_template: architecture: x86_64 type: linux distribution: ubuntu - version: 18.04 + version: 22.04 outputs: server_url: - value: { get_attribute: [ web_server, public_address ] } \ No newline at end of file + value: { get_attribute: [ web_server, public_address ] } diff --git a/test/files/tosca_remove_no_list.yml b/test/files/tosca_remove_no_list.yml index 17a6543c1..a00ac80b8 100644 --- a/test/files/tosca_remove_no_list.yml +++ b/test/files/tosca_remove_no_list.yml @@ -59,7 +59,7 @@ topology_template: # host Operating System image properties type: linux distribution: ubuntu - version: 16.04 + version: 22.04 test_db: type: tosca.nodes.indigo.Database.MySQL @@ -109,8 +109,8 @@ topology_template: architecture: x86_64 type: linux distribution: ubuntu - version: 18.04 + version: 22.04 outputs: server_url: - value: { get_attribute: [ web_server, public_address ] } \ No newline at end of file + value: { get_attribute: [ web_server, public_address ] } diff --git a/test/files/tosca_simple.yml b/test/files/tosca_simple.yml new file mode 100644 index 000000000..1310a412e --- /dev/null +++ b/test/files/tosca_simple.yml @@ -0,0 +1,25 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: TOSCA test for the IM + +topology_template: + + node_templates: + + server: + type: tosca.nodes.Compute + capabilities: + # Host container properties + host: + properties: + num_cpus: 1 + mem_size: 4 GB + os: + properties: + type: linux + distribution: ubuntu + version: 18.04 + + outputs: + server: + value: { get_attribute: [ server, public_address ] } \ No newline at end of file diff --git a/test/integration/TestIM.py b/test/integration/TestIM.py index 425523baf..ea0f711e3 100755 --- a/test/integration/TestIM.py +++ b/test/integration/TestIM.py @@ -435,7 +435,7 @@ def test_31_start(self): Test StartInfrastructure function """ # Assure the VM to be stopped - time.sleep(30) + time.sleep(60) (success, res) = self.server.StartInfrastructure( self.inf_id, self.auth_data) self.assertTrue( @@ -470,7 +470,7 @@ def test_33_start_vm(self): (success, vm_ids) = self.server.GetInfrastructureInfo( self.inf_id, self.auth_data) # Assure the VM to be stopped - time.sleep(30) + time.sleep(60) (success, res) = self.server.StartVM(self.inf_id, vm_ids[0], self.auth_data) self.assertTrue(success, msg="ERROR calling StartVM: " + str(res)) time.sleep(30) @@ -646,11 +646,11 @@ def test_80_create_ansible_host(self): host = master_radl.systems[0].getValue("net_interface.0.ip") username = master_radl.systems[0].getValue( "disk.0.os.credentials.username") - password = master_radl.systems[0].getValue( - "disk.0.os.credentials.password") + private_key = master_radl.systems[0].getValue( + "disk.0.os.credentials.private_key") radl = """ - ansible ansible_master (host = '%s' and credentials.username='%s' and credentials.password='%s') + ansible ansible_master (host = '%s' and credentials.username='%s' and credentials.private_key ='%s') network net () system node ( @@ -663,7 +663,7 @@ def test_80_create_ansible_host(self): ) deploy node 1 - """ % (host, username, password) + """ % (host, username, private_key) (success, inf_id) = self.server.CreateInfrastructure(radl, self.auth_data) self.assertTrue( diff --git a/test/integration/TestREST.py b/test/integration/TestREST.py index 64bc44fd6..de6f31816 100755 --- a/test/integration/TestREST.py +++ b/test/integration/TestREST.py @@ -376,7 +376,7 @@ def test_60_stop(self): def test_70_start(self): # To assure the VM is stopped - time.sleep(30) + time.sleep(60) resp = self.create_request("PUT", "/infrastructures/" + self.inf_id + "/start") self.assertEqual(resp.status_code, 200, msg="ERROR starting the infrastructure:" + resp.text) @@ -401,7 +401,7 @@ def test_80_stop_vm(self): def test_90_start_vm(self): # To assure the VM is stopped - time.sleep(30) + time.sleep(60) resp = self.create_request("PUT", "/infrastructures/" + self.inf_id + "/vms/0/start") self.assertEqual(resp.status_code, 200, msg="ERROR starting the vm:" + resp.text) diff --git a/test/unit/REST.py b/test/unit/REST.py index 5694abdfd..5d14a0cac 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -304,19 +304,20 @@ def test_CreateInfrastructure(self, bottle_request, get_infrastructure, CreateIn "id = one; type = OpenNebula; host = onedock.i3m.upv.es:2633; " "username = user; password = pass"), "Content-Type": "text/yaml"} - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.body = read_file_as_bytes("../files/tosca_simple.yml") CreateInfrastructure.return_value = "1" res = RESTCreateInfrastructure() self.assertEqual(res, "http://imserver.com/infrastructures/1") - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.headers["Content-Type"] = "application/json" + bottle_request.body = read_file_as_bytes("../files/test_simple.json") CreateInfrastructure.side_effect = InvaliddUserException() res = RESTCreateInfrastructure() self.assertEqual(res, "Error Getting Inf. info: Invalid InfrastructureManager credentials") - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.body = read_file_as_bytes("../files/test_simple.json") CreateInfrastructure.side_effect = UnauthorizedUserException() res = RESTCreateInfrastructure() self.assertEqual(res, "Error Creating Inf.: Access to this infrastructure not granted.") @@ -448,22 +449,23 @@ def test_AddResource(self, bottle_request, get_infrastructure, AddResource): "id = one; type = OpenNebula; host = onedock.i3m.upv.es:2633; " "username = user; password = pass"), "Content-Type": "text/yaml"} - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.body = read_file_as_bytes("../files/tosca_simple.yml") res = RESTAddResource("1") self.assertEqual(res, "http://imserver.com/infrastructures/1/vms/1") - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.headers["Content-Type"] = "application/json" + bottle_request.body = read_file_as_bytes("../files/test_simple.json") AddResource.side_effect = DeletedInfrastructureException() res = RESTAddResource("1") self.assertEqual(res, "Error Adding resources: Deleted infrastructure.") - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.body = read_file_as_bytes("../files/test_simple.json") AddResource.side_effect = IncorrectInfrastructureException() res = RESTAddResource("1") self.assertEqual(res, "Error Adding resources: Invalid infrastructure ID or access not granted.") - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.body = read_file_as_bytes("../files/test_simple.json") AddResource.side_effect = UnauthorizedUserException() res = RESTAddResource("1") self.assertEqual(res, "Error Adding resources: Access to this infrastructure not granted.") @@ -523,7 +525,7 @@ def test_AlterVM(self, bottle_request, AlterVM): "id = one; type = OpenNebula; host = onedock.i3m.upv.es:2633; " "username = user; password = pass"), "Content-Type": "text/yaml"} - bottle_request.body = read_file_as_bytes("../files/tosca_create.yml") + bottle_request.body = read_file_as_bytes("../files/tosca_simple.yml") res = RESTAlterVM("1", "1") self.assertEqual(res, "vm_info") diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index cf080877a..fa905865b 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -298,6 +298,7 @@ def test_tosca_oscar(self): self.assertEqual(node.getValue("memory.size"), 512000000) self.assertEqual(node.getValue("alpine"), 0) self.assertEqual(node.getValue("gpu.count"), 1) + self.assertEqual(node.getValue("cpu.sgx"), 1) self.assertEqual(node.getValue("input.0.path"), 'input') self.assertEqual(node.getValue("output.0.path"), 'output') self.assertEqual(node.getValue("onedata.0.id"), 'my_onedata') diff --git a/test/unit/connectors/OSCAR.py b/test/unit/connectors/OSCAR.py index 377cfb49c..9c4a4a83f 100755 --- a/test/unit/connectors/OSCAR.py +++ b/test/unit/connectors/OSCAR.py @@ -126,6 +126,7 @@ def test_20_launch(self, save_data, requests): name = 'plants' and memory.size = 2G and cpu.count = 1.0 and + cpu.sgx = 1 and gpu.count = 1 and disk.0.image.url = 'grycap/oscar-theano-plants' and script = 'plants.sh' and @@ -139,7 +140,11 @@ def test_20_launch(self, save_data, requests): minio.0.endpoint = 'https://minio.com' and minio.0.region = 'mregion' and minio.0.access_key = 'AK' and - minio.0.secret_key = 'SK' + minio.0.secret_key = 'SK' and + expose.min_scale = 1 and + expose.max_scale = 2 and + expose.port = 8080 and + expose.cpu_threshold = 70 )""" radl = radl_parse.parse_radl(radl_data) radl.check() @@ -160,6 +165,8 @@ def test_20_launch(self, save_data, requests): expected_res = {"name": "plants", "memory": "2048Mi", "cpu": "1", "script": "plants.sh", "enable_gpu": True, + "enable_sgx": True, + "expose": {"cpu_threshold": 70, "max_scale": 2, "min_scale": 1, "port": 8080}, "image": "grycap/oscar-theano-plants", "environment": {"Variables": {"a": "b", "VAR": "https://server"}}, "input": [{"storage_provider": "minio_id", diff --git a/test/unit/db.py b/test/unit/db.py index 51eedb435..97cc66ce4 100644 --- a/test/unit/db.py +++ b/test/unit/db.py @@ -88,6 +88,10 @@ def test_mongo_db(self, mongo): self.assertTrue(res) self.assertEqual(table.replace_one.call_args_list[0][0], ({}, {'data': 'test1', 'id': 1}, True)) + res = db.update('table', {'id': 1}, {'data': 'test1'}) + self.assertTrue(res) + self.assertEqual(table.update_one.call_args_list[0][0], ({'id': 1}, {'data': 'test1'}, True)) + table.find.return_value = [{'id': 2, 'data': 'test2', '_id': 2}] res = db.find('table', {'id': 2}, {'data': True}) self.assertEqual(len(res), 1)