diff --git a/IM/InfrastructureInfo.py b/IM/InfrastructureInfo.py index 4b2ad0b10..45f594c99 100644 --- a/IM/InfrastructureInfo.py +++ b/IM/InfrastructureInfo.py @@ -361,12 +361,12 @@ def update_radl(self, radl, deployed_vms, warn=True): # Add new networks ad ansible_hosts only for s in radl.networks + radl.ansible_hosts: if not self.radl.add(s.clone(), "ignore") and warn: - InfrastructureInfo.logger.warn("Ignoring the redefinition of %s %s" % (type(s), s.getId())) + InfrastructureInfo.logger.warning("Ignoring the redefinition of %s %s" % (type(s), s.getId())) # Add or update configures and systems for s in radl.configures + radl.systems: if self.radl.get(s) and warn: - InfrastructureInfo.logger.warn("(Re)definition of %s %s" % (type(s), s.getId())) + InfrastructureInfo.logger.warning("(Re)definition of %s %s" % (type(s), s.getId())) self.radl.add(s.clone(), "replace") # Append contextualize @@ -724,7 +724,7 @@ def _is_authorized(self, self_im_auth, auth): if (self_im_auth['username'].startswith(InfrastructureInfo.OPENID_USER_PREFIX) and 'token' not in other_im_auth): # This is a OpenID user do not enable to get data using user/pass creds - InfrastructureInfo.logger.warn("Inf ID %s: A non OpenID user tried to access it." % self.id) + InfrastructureInfo.logger.warning("Inf ID %s: A non OpenID user tried to access it." % self.id) res = False break diff --git a/IM/InfrastructureList.py b/IM/InfrastructureList.py index 8c8b371df..e24da4ff9 100644 --- a/IM/InfrastructureList.py +++ b/IM/InfrastructureList.py @@ -210,7 +210,7 @@ def _get_data_from_db(db_url, inf_id=None, auth=None): msg = "" if inf_id: msg = " for inf ID: %s" % inf_id - InfrastructureList.logger.warn("No data in database%s!." % msg) + InfrastructureList.logger.warning("No data in database%s!." % msg) db.close() return inf_list diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index ee17e6834..ce043afb8 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -518,6 +518,53 @@ def sort_by_score(sel_inf, concrete_systems, cloud_list, deploy_groups, auth): return deploys_group_cloud + @staticmethod + def add_app_reqs(radl, inf_id=None): + """ Add apps requirements to the RADL. """ + for radl_system in radl.systems: + apps_to_install = radl_system.getApplications() + for app_to_install in apps_to_install: + for app_avail, _, _, _, requirements in Recipe.getInstallableApps(): + if requirements and app_avail.isNewerThan(app_to_install): + # This app must be installed and it has special + # requirements + try: + requirements_radl = radl_parse.parse_radl(requirements).systems[0] + radl_system.applyFeatures(requirements_radl, conflict="other", missing="other") + except Exception: + InfrastructureManager.logger.exception( + "Inf ID: " + inf_id + ": Error in the requirements of the app: " + + app_to_install.getValue("name") + ". Ignore them.") + InfrastructureManager.logger.debug("Inf ID: " + inf_id + ": " + str(requirements)) + break + + @staticmethod + def get_deploy_groups(cloud_list, radl, systems_with_iis, sel_inf, auth): + # Concrete systems with cloud providers and select systems with the greatest score + # in every cloud + concrete_systems = {} + for cloud_id, cloud in cloud_list.items(): + for system_id, systems in systems_with_iis.items(): + s1 = [InfrastructureManager._compute_score(s.clone().applyFeatures(s0, + conflict="other", + missing="other").concrete(), + radl.get_system_by_name(system_id)) + for s in systems for s0 in cloud.concreteSystem(s, auth)] + # Store the concrete system with largest score + concrete_systems.setdefault(cloud_id, {})[system_id] = max(s1, key=lambda x: x[1]) if s1 else (None, + -1e9) + + # Group virtual machines to deploy by network dependencies + deploy_groups = InfrastructureManager._compute_deploy_groups(radl) + InfrastructureManager.logger.debug("Inf ID: " + sel_inf.id + ": Groups of VMs with dependencies") + InfrastructureManager.logger.debug("Inf ID: " + sel_inf.id + "\n" + str(deploy_groups)) + + # Sort by score the cloud providers + deploys_group_cloud = InfrastructureManager.sort_by_score(sel_inf, concrete_systems, cloud_list, + deploy_groups, auth) + + return concrete_systems, deploy_groups, deploys_group_cloud + @staticmethod def AddResource(inf_id, radl_data, auth, context=True): """ @@ -560,7 +607,7 @@ def AddResource(inf_id, radl_data, auth, context=True): # If any deploy is defined, only update definitions. if not radl.deploys: - InfrastructureManager.logger.warn("Inf ID: " + sel_inf.id + ": without any deploy. Exiting.") + InfrastructureManager.logger.warning("Inf ID: " + sel_inf.id + ": without any deploy. Exiting.") sel_inf.add_cont_msg("Infrastructure without any deploy. Exiting.") if sel_inf.configured is None: sel_inf.configured = False @@ -571,23 +618,7 @@ def AddResource(inf_id, radl_data, auth, context=True): InfrastructureManager.logger.exception("Inf ID: " + sel_inf.id + " error parsing RADL") raise ex - for radl_system in radl.systems: - # Add apps requirements to the RADL - apps_to_install = radl_system.getApplications() - for app_to_install in apps_to_install: - for app_avail, _, _, _, requirements in Recipe.getInstallableApps(): - if requirements and app_avail.isNewerThan(app_to_install): - # This app must be installed and it has special - # requirements - try: - requirements_radl = radl_parse.parse_radl(requirements).systems[0] - radl_system.applyFeatures(requirements_radl, conflict="other", missing="other") - except Exception: - InfrastructureManager.logger.exception( - "Inf ID: " + sel_inf.id + ": Error in the requirements of the app: " + - app_to_install.getValue("name") + ". Ignore them.") - InfrastructureManager.logger.debug("Inf ID: " + sel_inf.id + ": " + str(requirements)) - break + InfrastructureManager.add_app_reqs(radl, sel_inf.id) # Concrete systems using VMRC try: @@ -601,26 +632,11 @@ def AddResource(inf_id, radl_data, auth, context=True): # Concrete systems with cloud providers and select systems with the greatest score # in every cloud cloud_list = dict([(c.id, c.getCloudConnector(sel_inf)) for c in CloudInfo.get_cloud_list(auth)]) - concrete_systems = {} - for cloud_id, cloud in cloud_list.items(): - for system_id, systems in systems_with_iis.items(): - s1 = [InfrastructureManager._compute_score(s.clone().applyFeatures(s0, - conflict="other", - missing="other").concrete(), - radl.get_system_by_name(system_id)) - for s in systems for s0 in cloud.concreteSystem(s, auth)] - # Store the concrete system with largest score - concrete_systems.setdefault(cloud_id, {})[system_id] = ( - max(s1, key=lambda x: x[1]) if s1 else (None, -1e9)) - - # Group virtual machines to deploy by network dependencies - deploy_groups = InfrastructureManager._compute_deploy_groups(radl) - InfrastructureManager.logger.debug("Inf ID: " + sel_inf.id + ": Groups of VMs with dependencies") - InfrastructureManager.logger.debug("Inf ID: " + sel_inf.id + "\n" + str(deploy_groups)) - - # Sort by score the cloud providers - deploys_group_cloud = InfrastructureManager.sort_by_score(sel_inf, concrete_systems, cloud_list, - deploy_groups, auth) + concrete_systems, deploy_groups, deploys_group_cloud = InfrastructureManager.get_deploy_groups(cloud_list, + radl, + systems_with_iis, + sel_inf, + auth) # We are going to start adding resources sel_inf.set_adding() @@ -1409,6 +1425,18 @@ def check_oidc_token(im_auth): if not issuer.endswith('/'): issuer += '/' im_auth['password'] = issuer + str(userinfo.get("sub")) + + if Config.OIDC_GROUPS: + # Get user groups from any of the possible fields + user_groups = userinfo.get('groups', # Generic + userinfo.get('entitlements', # GEANT + userinfo.get('eduperson_entitlement', # EGI Check-in + []))) + + if not set(Config.OIDC_GROUPS).issubset(user_groups): + raise InvaliddUserException("Invalid InfrastructureManager credentials. " + + "User not in configured groups.") + except Exception as ex: InfrastructureManager.logger.exception("Error trying to validate OIDC auth token: %s" % str(ex)) raise Exception("Error trying to validate OIDC auth token: %s" % str(ex)) @@ -1526,13 +1554,13 @@ def check_auth_data(auth): if im_auth_item['username'].startswith(IM.InfrastructureInfo.InfrastructureInfo.OPENID_USER_PREFIX): # This is a OpenID user do not enable to get data using user/pass creds raise InvaliddUserException("Invalid username used for the InfrastructureManager.") + + # Now check if the user is in authorized + if not InfrastructureManager.check_im_user(im_auth_item): + raise InvaliddUserException() else: raise InvaliddUserException("No username nor token for the InfrastructureManager.") - # Now check if the user is in authorized - if not InfrastructureManager.check_im_user(im_auth_item): - raise InvaliddUserException() - if Config.SINGLE_SITE: vmrc_auth = auth.getAuthInfo("VMRC") single_site_auth = auth.getAuthInfo(Config.SINGLE_SITE_TYPE) @@ -1869,3 +1897,130 @@ def GetInfrastructureOwners(inf_id, auth): res.append(im_auth['username']) return res + + @staticmethod + def EstimateResouces(radl_data, auth): + """ + Get the estimated amount of resources needed to deploy the infrastructure. + + Args: + + - radl(str): RADL description. + - auth(Authentication): parsed authentication tokens. + + Return(dict): dict with the estimated amount of needed to deploy the infrastructure + with the following structure: + { + "ost1": { + "cloudType": "OpenStack", + "cloudEndpoint": "http://openstack.example.com:5000", + + "compute": [ + { + "cpuCores": 2, + "memoryInMegabytes": 4096, + "diskSizeInGigabytes": 20 + }, + { + "cpuCores": 1, + "memoryInMegabytes": 2048, + "diskSizeInGigabytes": 10 + } + ], + "storage": [ + {"sizeInGigabytes": 100, "type": "ceph"}, + {"sizeInGigabytes": 100} + ] + } + } + + """ + res = {} + InfrastructureManager.logger.info("Getting the cost of the infrastructure") + + # First check the auth data + auth = InfrastructureManager.check_auth_data(auth) + + try: + if isinstance(radl_data, RADL): + radl = radl_data + else: + radl = radl_parse.parse_radl(radl_data) + + radl.check() + + inf = IM.InfrastructureInfo.InfrastructureInfo() + inf.radl = radl + + # If any deploy is defined, only update definitions. + if not radl.deploys: + InfrastructureManager.logger.warning("Getting cost of an infrastructure without any deploy.") + return res + except Exception as ex: + InfrastructureManager.logger.exception("Error getting cost of an infrastructure when parsing RADL") + raise ex + + InfrastructureManager.add_app_reqs(radl, inf.id) + + # Concrete systems using VMRC + try: + systems_with_iis = InfrastructureManager.systems_with_iis(inf, radl, auth) + except Exception as ex: + InfrastructureManager.logger.exception("Error getting cost of an infrastructure error getting VM images") + raise ex + + # Concrete systems with cloud providers and select systems with the greatest score + # in every cloud + cloud_list = dict([(c.id, c.getCloudConnector(inf)) for c in CloudInfo.get_cloud_list(auth)]) + concrete_systems, deploy_groups, deploys_group_cloud = InfrastructureManager.get_deploy_groups(cloud_list, + radl, + systems_with_iis, + inf, + auth) + + # Launch every group in the same cloud provider + deploy_items = [] + for deploy_group in deploy_groups: + if not deploy_group: + InfrastructureManager.logger.warning("Error getting cost of an infrastructure: No VMs to deploy!") + + cloud_id = deploys_group_cloud[id(deploy_group)] + cloud = cloud_list[cloud_id] + + for d in deploy_group: + deploy_items.append((d, cloud_id, cloud)) + + # Get the cost of the infrastructure + for deploy, cloud_id, cloud in deploy_items: + + if cloud_id not in res: + res[cloud_id] = {"cloudType": cloud.type, "cloudEndpoint": cloud.cloud.get_url(), + "compute": [], "storage": []} + + if not deploy.id.startswith(IM.InfrastructureInfo.InfrastructureInfo.FAKE_SYSTEM): + concrete_system = concrete_systems[cloud_id][deploy.id][0] + + if not concrete_system: + InfrastructureManager.logger.warning("Error getting cost of an infrastructure:" + + "Error, no concrete system to deploy: " + + deploy.id + " in cloud: " + cloud_id + + ". Check if a correct image is being used") + else: + vm = {"cpuCores": concrete_system.getValue("cpu.count"), + "memoryInMegabytes": concrete_system.getFeature("memory.size").getValue("M")} + + if concrete_system.getValue("disk.0.free_size"): + vm['diskSizeInGigabytes'] = concrete_system.getFeature("disk.0.free_size").getValue('G') + + res[cloud_id]["compute"].append(vm) + + cont = 1 + while (concrete_system.getValue("disk." + str(cont) + ".size")): + volume_size = concrete_system.getFeature("disk." + str(cont) + ".size").getValue('G') + vol_info = {"sizeInGigabytes": volume_size} + if concrete_system.getValue("disk." + str(cont) + ".type"): + vol_info["type"] = concrete_system.getValue("disk." + str(cont) + ".type") + res[cloud_id]["storage"].append(vol_info) + cont += 1 + + return res diff --git a/IM/REST.py b/IM/REST.py index 1c1eca247..571bc433f 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -534,14 +534,24 @@ def RESTCreateInfrastructure(): async_call = False if "async" in bottle.request.params.keys(): - str_ctxt = bottle.request.params.get("async").lower() - if str_ctxt in ['yes', 'true', '1']: + str_async = bottle.request.params.get("async").lower() + if str_async in ['yes', 'true', '1']: async_call = True - elif str_ctxt in ['no', 'false', '0']: + elif str_async in ['no', 'false', '0']: async_call = False else: return return_error(400, "Incorrect value in async parameter") + dry_run = False + if "dry_run" in bottle.request.params.keys(): + str_dry_run = bottle.request.params.get("dry_run").lower() + if str_dry_run in ['yes', 'true', '1']: + dry_run = True + elif str_dry_run in ['no', 'false', '0']: + dry_run = False + else: + return return_error(400, "Incorrect value in dry_run parameter") + if content_type: if "application/json" in content_type: radl_data = parse_radl_json(radl_data) @@ -553,18 +563,22 @@ def RESTCreateInfrastructure(): else: return return_error(415, "Unsupported Media Type %s" % content_type) - inf_id = InfrastructureManager.CreateInfrastructure(radl_data, auth, async_call) + if dry_run: + res = InfrastructureManager.EstimateResouces(radl_data, auth) + return format_output(res, "application/json") + else: + inf_id = InfrastructureManager.CreateInfrastructure(radl_data, auth, async_call) - # Store the TOSCA document - if tosca_data: - sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) - sel_inf.extra_info['TOSCA'] = tosca_data + # Store the TOSCA document + if tosca_data: + sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) + sel_inf.extra_info['TOSCA'] = tosca_data - bottle.response.headers['InfID'] = inf_id - bottle.response.content_type = "text/uri-list" - res = get_full_url('/infrastructures/%s' % inf_id) + bottle.response.headers['InfID'] = inf_id + bottle.response.content_type = "text/uri-list" + res = get_full_url('/infrastructures/%s' % inf_id) - return format_output(res, "text/uri-list", "uri") + return format_output(res, "text/uri-list", "uri") except InvaliddUserException as ex: return return_error(401, "Error Getting Inf. info: %s" % get_ex_error(ex)) except DisabledFunctionException as ex: @@ -685,7 +699,7 @@ def RESTGetVMProperty(infid=None, vmid=None, prop=None): break if not sel_vm: # it sometimes happen when the VM is in creation state - logger.warn("Specified vmid in step2 is incorrect!!") + logger.warning("Specified vmid in step2 is incorrect!!") info = "wait" else: ssh = sel_vm.get_ssh_ansible_master(retry=False) diff --git a/IM/ServiceRequests.py b/IM/ServiceRequests.py index 501349d91..0f8cfad19 100644 --- a/IM/ServiceRequests.py +++ b/IM/ServiceRequests.py @@ -59,6 +59,7 @@ class IMBaseRequest(AsyncRequest): GET_CLOUD_QUOTAS = "GetCloudQuotas" CHANGE_INFRASTRUCTURE_AUTH = "ChangeInfrastructureAuth" GET_INFRASTRUCTURE_OWNERS = "GetInfrastructureOwners" + ESTIMATE_RESOURCES = "EstimateResouces" @staticmethod def create_request(function, arguments=()): @@ -116,6 +117,8 @@ def create_request(function, arguments=()): return Request_ChangeInfrastructureAuth(arguments) elif function == IMBaseRequest.GET_INFRASTRUCTURE_OWNERS: return Request_GetInfrastructureOwners(arguments) + elif function == IMBaseRequest.ESTIMATE_RESOURCES: + return Request_EstimateResouces(arguments) else: raise NotImplementedError("Function not Implemented") @@ -459,3 +462,14 @@ def _call_function(self): self._error_mesage = "Error getting the Inf. owners" (inf_id, auth_data) = self.arguments return IM.InfrastructureManager.InfrastructureManager.GetInfrastructureOwners(inf_id, Authentication(auth_data)) + + +class Request_EstimateResouces(IMBaseRequest): + """ + Request class for the EstimateResouces function + """ + + def _call_function(self): + self._error_mesage = "Error getting the resources estimation" + (radl_data, auth_data) = self.arguments + return IM.InfrastructureManager.InfrastructureManager.EstimateResouces(radl_data, Authentication(auth_data)) diff --git a/IM/config.py b/IM/config.py index 18249f272..0429ae947 100644 --- a/IM/config.py +++ b/IM/config.py @@ -42,8 +42,7 @@ def parse_options(config, section_name, config_class): setattr(config_class, option, config.get(section_name, option)) else: logger = logging.getLogger('InfrastructureManager') - logger.warn( - "Unknown option in the IM config file. Ignoring it: " + option) + logger.warning("Unknown option in the IM config file. Ignoring it: " + option) class Config: @@ -107,6 +106,7 @@ class Config: OIDC_SCOPES = [] OIDC_USER_INFO_PATH = "/userinfo" OIDC_INSTROSPECT_PATH = "/introspect" + OIDC_GROUPS = [] VM_NUM_USE_CTXT_DIST = 30 DELAY_BETWEEN_VM_RETRIES = 5 VERIFI_SSL = False diff --git a/IM/connectors/OpenNebula.py b/IM/connectors/OpenNebula.py index a3c84e363..06dc97610 100644 --- a/IM/connectors/OpenNebula.py +++ b/IM/connectors/OpenNebula.py @@ -1246,7 +1246,7 @@ def delete_image(self, image_url, auth_data): # Wait the image to be READY (not USED) success, msg = self.wait_image(image_id, auth_data) if not success: - self.logger.warn("Error waiting image to be READY: " + msg) + self.logger.warning("Error waiting image to be READY: " + msg) success, res_info = server.one.image.delete(session_id, image_id)[0:2] if success: diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index 135a3d8b7..c2e738827 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -494,6 +494,9 @@ def setVolumesInfo(self, vm, node): if 'attachments' in volume.extra and volume.extra['attachments']: vm.info.systems[0].setValue("disk." + str(cont) + ".device", os.path.basename(volume.extra['attachments'][0]['device'])) + if 'volume_type' in volume.extra and volume.extra['volume_type']: + vm.info.systems[0].setValue("disk." + str(cont) + ".type", volume.extra['volume_type']) + cont += 1 except Exception as ex: self.log_warn("Error getting volume info: %s" % get_ex_error(ex)) diff --git a/contextualization/ansible_install.sh b/contextualization/ansible_install.sh index 0ad3fce6a..fcdcbb3f1 100755 --- a/contextualization/ansible_install.sh +++ b/contextualization/ansible_install.sh @@ -76,7 +76,7 @@ else fi # Create the config file -mkdir /etc/ansible +ls /etc/ansible || mkdir /etc/ansible cat > /etc/ansible/ansible.cfg <`_). + The default value is ``''``. + .. confval:: FORCE_OIDC_AUTH If ``True`` the IM will force the users to pass a valid OIDC token. diff --git a/doc/source/xmlrpc.rst b/doc/source/xmlrpc.rst index 9e65562cf..3bbb37c38 100644 --- a/doc/source/xmlrpc.rst +++ b/doc/source/xmlrpc.rst @@ -416,3 +416,39 @@ This is the list of method names: :fail response: [false, ``error``: string] Return the list of current owners of the infrastructure with ID ``infId``. + +.. _EstimateResouces-xmlrpc: + +``EstimateResouces`` + :parameter 0: ``radl``: string + :parameter 1: ``auth``: array of structs + :ok response: [true, struct] + :fail response: [false, ``error``: string] + + Get the estimated amount of resources needed to deploy the infrastructure + specified in the RADL document passed as string. The response is a struct + with the following format (memory unit MB, disk and storage unit GB):: + + { + "ost1": { + "cloudType": "OpenStack", + "cloudEndpoint": "http://openstack.example.com:5000", + + "compute": [ + { + "cpuCores": 2, + "memoryInMegabytes": 4096, + "diskSizeInGigabytes": 20 + }, + { + "cpuCores": 1, + "memoryInMegabytes": 2048, + "diskSizeInGigabytes": 10 + } + ], + "storage": [ + {"sizeInGigabytes": 100, "type": "ceph"}, + {"sizeInGigabytes": 100} + ] + } + } diff --git a/doc/swagger_api.yaml b/doc/swagger_api.yaml index 04fbbc390..29cd84253 100644 --- a/doc/swagger_api.yaml +++ b/doc/swagger_api.yaml @@ -2,7 +2,7 @@ openapi: 3.0.0 info: description: Infrastructure Manager (IM) REST API. - version: 1.11.0 + version: 1.17.0 title: Infrastructure Manager (IM) REST API contact: email: products@grycap.upv.es @@ -111,6 +111,22 @@ paths: - '0' - '1' default: 'false' + - name: dry_run + in: query + description: >- + parameter is optional and is a flag to specify if the call will not create the VMs + and will only return the ammount of resources needed to deploy the infrastructure. + required: false + schema: + type: string + enum: + - 'yes' + - 'no' + - 'true' + - 'false' + - '0' + - '1' + default: 'false' requestBody: content: text/plain: @@ -177,6 +193,17 @@ paths: { "uri": "http://server.com:8800/infrastructures/inf_id1" } + response_dry_run: + value: | + { + "compute": [ + {"cpu": 2, "memory": 4096, "disk": 20}, + {"cpu": 1, "memory": 2048, "disk": 10} + ], + "storage": [ + {"size": 100} + ] + } '400': description: Invalid status value '401': diff --git a/etc/im.cfg b/etc/im.cfg index a6525ae4f..be1b5b90a 100644 --- a/etc/im.cfg +++ b/etc/im.cfg @@ -97,7 +97,7 @@ UPDATE_CTXT_LOG_INTERVAL = 20 # Interval to update the state of the processes of the ConfManager (in secs) CONFMAMAGER_CHECK_STATE_INTERVAL = 5 # Max time expected to install Ansible in the master node -ANSIBLE_INSTALL_TIMEOUT = 500 +ANSIBLE_INSTALL_TIMEOUT = 900 # Number of VMs in an infrastructure that will use the distributed version of the Ctxt Agent VM_NUM_USE_CTXT_DIST = 30 @@ -146,6 +146,8 @@ OIDC_ISSUERS = https://aai.egi.eu/auth/realms/egi # Paths to the userinfo and introspection OIDC #OIDC_USER_INFO_PATH = "/userinfo" #OIDC_INSTROSPECT_PATH = "/introspect" +# List of OIDC groups that will be allowed to access the IM service +#OIDC_GROUPS = # Force the users to pass a valid OIDC token #FORCE_OIDC_AUTH = False diff --git a/im_service.py b/im_service.py index bb5ae135a..a242e56ea 100755 --- a/im_service.py +++ b/im_service.py @@ -226,6 +226,12 @@ def GetInfrastructureOwners(inf_id, auth_data): return WaitRequest(request) +def EstimateResources(radl_data, auth_data): + request = IMBaseRequest.create_request( + IMBaseRequest.ESTIMATE_RESOURCES, (radl_data, auth_data)) + return WaitRequest(request) + + def launch_daemon(): """ Launch the IM daemon @@ -286,6 +292,7 @@ def launch_daemon(): server.register_function(GetCloudQuotas) server.register_function(ChangeInfrastructureAuth) server.register_function(GetInfrastructureOwners) + server.register_function(EstimateResources) # Launch the API XMLRPC thread server.serve_forever_in_thread() diff --git a/test/files/iam_user_info.json b/test/files/iam_user_info.json index 7bc8b105c..848fde0ea 100644 --- a/test/files/iam_user_info.json +++ b/test/files/iam_user_info.json @@ -6,15 +6,10 @@ "email": "", "email_verified": true, "phone_number_verified": false, - "groups": [ - { - "id": "gid", - "name": "Users" - }, - { - "id": "gid", - "name": "Developers" - } + "eduperson_entitlement": [ + "urn:mace:egi.eu:group:demo.fedcloud.egi.eu:members:role=member#aai.egi.eu", + "urn:mace:egi.eu:group:demo.fedcloud.egi.eu:role=member#aai.egi.eu", + "urn:mace:egi.eu:group:demo.fedcloud.egi.eu:vm_operator:role=member#aai.egi.eu" ], "organisation_name": "indigo-dc" } \ No newline at end of file diff --git a/test/files/quick-test.radl b/test/files/quick-test.radl index ac15cc055..8297629a5 100644 --- a/test/files/quick-test.radl +++ b/test/files/quick-test.radl @@ -24,7 +24,7 @@ cpu.count>=1 and memory.size>=512m and net_interface.0.connection = 'privada' and disk.0.os.name='linux' and -disk.0.image.url = ['one://ramses.i3m.upv.es/1396', 'ost://horsemen.i3m.upv.es/609f8280-fbb6-46bd-84e2-5315b22414f1'] and +disk.0.image.url = ['one://ramses.i3m.upv.es/1593', 'ost://horsemen.i3m.upv.es/79095fa5-7baf-493c-9514-e24d52dd1527'] and disk.1.size=1GB and disk.1.device='hdb' and disk.1.fstype='ext4' and diff --git a/test/files/reverse.radl b/test/files/reverse.radl index ef413135c..4d4eb5bef 100644 --- a/test/files/reverse.radl +++ b/test/files/reverse.radl @@ -27,7 +27,7 @@ memory.size>=1024m and net_interface.0.connection = 'privada2' and disk.0.os.name='linux' and disk.0.os.flavour='ubuntu' and -disk.0.image.url = 'one://ramses.i3m.upv.es/1396' +disk.0.image.url = 'one://ramses.i3m.upv.es/1593' ) diff --git a/test/files/test_ansible.radl b/test/files/test_ansible.radl index cfc9fa5ea..5fa03d9f8 100644 --- a/test/files/test_ansible.radl +++ b/test/files/test_ansible.radl @@ -6,7 +6,7 @@ cpu.count>=1 and memory.size>=1g and net_interface.0.connection = 'publica' and disk.0.os.name='linux' and -disk.0.image.url = 'one://ramses.i3m.upv.es/1396' +disk.0.image.url = 'one://ramses.i3m.upv.es/1593' ) configure node ( diff --git a/test/files/test_simple.json b/test/files/test_simple.json index d8d5f128f..c12648d99 100644 --- a/test/files/test_simple.json +++ b/test/files/test_simple.json @@ -12,12 +12,10 @@ "class": "system", "cpu.arch": "x86_64", "cpu.count_min": 1, - "disk.0.image.url": "one://ramses.i3m.upv.es/1396", - "disk.0.os.credentials.password": "GRyCAP01", - "disk.0.os.credentials.username": "ubuntu", + "disk.0.image.url": "one://ramses.i3m.upv.es/1593", "disk.0.os.name": "linux", "id": "front", - "memory.size_min": 536870912, + "memory.size_min": 1073741824, "net_interface.0.connection": "publica", "net_interface.1.connection": "privada" }, @@ -25,10 +23,10 @@ "class": "system", "cpu.arch": "x86_64", "cpu.count_min": 1, - "disk.0.image.url": "one://ramses.i3m.upv.es/1129", + "disk.0.image.url": "one://ramses.i3m.upv.es/1396", "disk.0.os.name": "linux", "id": "wn", - "memory.size_min": 536870912, + "memory.size_min": 1073741824, "net_interface.0.connection": "privada" }, { diff --git a/test/files/tosca_ansible_host.yaml b/test/files/tosca_ansible_host.yaml index b471cd5eb..e30981a84 100644 --- a/test/files/tosca_ansible_host.yaml +++ b/test/files/tosca_ansible_host.yaml @@ -56,7 +56,7 @@ topology_template: mem_size: 1 GB os: properties: - image: 'one://ramses.i3m.upv.es/1396' + image: 'one://ramses.i3m.upv.es/1593' type: linux credential: token_type: password # or private_key diff --git a/test/integration/TestIM.py b/test/integration/TestIM.py index 9ad7b1ce9..da9ef55ac 100755 --- a/test/integration/TestIM.py +++ b/test/integration/TestIM.py @@ -272,7 +272,7 @@ def test_19_addresource(self): str(len(vm_ids)) + "). It must be 4")) all_configured = self.wait_inf_state( - self.inf_id, VirtualMachine.CONFIGURED, 1500) + self.inf_id, VirtualMachine.CONFIGURED, 2100) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") @@ -337,7 +337,7 @@ def test_22_removeresource(self): msg="ERROR unexpected state. Expected 'running' and obtained " + vm_state) all_configured = self.wait_inf_state( - self.inf_id, VirtualMachine.CONFIGURED, 1200) + self.inf_id, VirtualMachine.CONFIGURED, 1800) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") @@ -635,7 +635,7 @@ def test_80_create_ansible_host(self): self.__class__.inf_id = [inf_id] all_configured = self.wait_inf_state( - inf_id, VirtualMachine.CONFIGURED, 900) + inf_id, VirtualMachine.CONFIGURED, 1200) self.assertTrue( all_configured, msg="ERROR waiting the ansible master to be configured (timeout).") @@ -672,7 +672,7 @@ def test_80_create_ansible_host(self): self.__class__.inf_id.append(inf_id) all_configured = self.wait_inf_state( - inf_id, VirtualMachine.CONFIGURED, 900) + inf_id, VirtualMachine.CONFIGURED, 1200) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") @@ -698,7 +698,7 @@ def test_90_create(self): self.__class__.inf_id = [inf_id] all_configured = self.wait_inf_state( - inf_id, VirtualMachine.CONFIGURED, 1800) + inf_id, VirtualMachine.CONFIGURED, 2100) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") @@ -723,7 +723,7 @@ def test_96_create(self): success, msg="ERROR calling CreateInfrastructure: " + str(inf_id)) self.__class__.inf_id = [inf_id] - all_configured = self.wait_inf_state(inf_id, VirtualMachine.CONFIGURED, 900) + all_configured = self.wait_inf_state(inf_id, VirtualMachine.CONFIGURED, 1200) self.assertTrue( all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") @@ -759,7 +759,7 @@ def test_98_proxy(self): self.assertTrue(success, msg="ERROR calling CreateInfrastructure: " + str(inf_id)) self.__class__.inf_id = [inf_id] - all_configured = self.wait_inf_state(inf_id, VirtualMachine.CONFIGURED, 900) + all_configured = self.wait_inf_state(inf_id, VirtualMachine.CONFIGURED, 1200) self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") (success, vminfo) = self.server.GetVMInfo(inf_id, 0, self.auth_data) @@ -778,7 +778,7 @@ def test_98_proxy(self): memory.size>=1g and net_interface.0.connection = 'net' and disk.0.os.name='linux' and - disk.0.image.url = 'one://ramses.i3m.upv.es/1396' + disk.0.image.url = 'one://ramses.i3m.upv.es/1593' ) deploy test 1 @@ -786,7 +786,7 @@ def test_98_proxy(self): (success, inf_id2) = self.server.CreateInfrastructure(radl, self.auth_data) self.__class__.inf_id.append(inf_id2) - all_configured = self.wait_inf_state(inf_id2, VirtualMachine.CONFIGURED, 900) + all_configured = self.wait_inf_state(inf_id2, VirtualMachine.CONFIGURED, 1200) self.assertTrue(all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).") def test_99_destroy(self): diff --git a/test/unit/REST.py b/test/unit/REST.py index d5354a622..4b3ac4ec7 100755 --- a/test/unit/REST.py +++ b/test/unit/REST.py @@ -323,6 +323,19 @@ def test_CreateInfrastructure(self, bottle_request, get_infrastructure, CreateIn res = RESTCreateInfrastructure() self.assertEqual(res, "Error Creating Inf.: Access to this infrastructure not granted.") + # Test the dry_run option to get the estimation of the resources + bottle_request.headers = {"AUTHORIZATION": ("type = InfrastructureManager; username = user; password = pass\\n" + "id = one; type = OpenNebula; host = ramses.i3m.upv.es:2633; " + "username = user; password = pass"), + "Content-Type": "application/json"} + bottle_request.body = read_file_as_bytes("../files/test_simple.json") + bottle_request.params = {"dry_run": "yes"} + res = RESTCreateInfrastructure() + self.assertEqual(res, ('{"one": {"cloudType": "OpenNebula", "cloudEndpoint":' + ' "http://ramses.i3m.upv.es:2633",' + ' "compute": [{"cpuCores": 1, "memoryInMegabytes": 1024},' + ' {"cpuCores": 1, "memoryInMegabytes": 1024}], "storage": []}}')) + @patch("IM.InfrastructureManager.InfrastructureManager.CreateInfrastructure") @patch("bottle.request") def test_CreateInfrastructureWithErrors(self, bottle_request, CreateInfrastructure): diff --git a/test/unit/ServiceRequests.py b/test/unit/ServiceRequests.py index d865147f5..4bf16b980 100755 --- a/test/unit/ServiceRequests.py +++ b/test/unit/ServiceRequests.py @@ -207,6 +207,13 @@ def test_get_owners(self, inflist): IM.ServiceRequests.IMBaseRequest.GET_INFRASTRUCTURE_OWNERS, ("", "")) req._call_function() + @patch('IM.InfrastructureManager.InfrastructureManager') + def test_estimate_resources(self, inflist): + import IM.ServiceRequests + req = IM.ServiceRequests.IMBaseRequest.create_request( + IM.ServiceRequests.IMBaseRequest.ESTIMATE_RESOURCES, ("", "")) + req._call_function() + if __name__ == '__main__': unittest.main() diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py index 7450f4aa4..c2389b382 100644 --- a/test/unit/test_im_logic.py +++ b/test/unit/test_im_logic.py @@ -135,7 +135,7 @@ def get_cloud_connector_mock(self, name="MyMock0"): return cloud @staticmethod - def gen_token(aud=None, exp=None, user_sub="user_sub"): + def gen_token(aud=None, exp=None, user_sub="user_sub", groups=None): data = { "sub": user_sub, "iss": "https://iam-test.indigo-datacloud.eu/", @@ -147,6 +147,8 @@ def gen_token(aud=None, exp=None, user_sub="user_sub"): data["aud"] = aud if exp: data["exp"] = int(time.time()) + exp + if groups: + data["groups"] = groups return ("eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.%s.ignored" % base64.urlsafe_b64encode(json.dumps(data).encode("utf-8")).decode("utf-8")) @@ -1126,6 +1128,34 @@ def test_check_oidc_valid_token(self, openidclient): self.assertEqual(im_auth['username'], InfrastructureInfo.OPENID_USER_PREFIX + "micafer") self.assertEqual(im_auth['password'], "https://iam-test.indigo-datacloud.eu/sub") + @patch('IM.InfrastructureManager.OpenIDClient') + def test_check_oidc_groups(self, openidclient): + im_auth = {"token": (self.gen_token())} + + user_info = json.loads(read_file_as_string('../files/iam_user_info.json')) + + openidclient.is_access_token_expired.return_value = False, "Valid Token for 100 seconds" + openidclient.get_user_info_request.return_value = True, user_info + + Config.OIDC_ISSUERS = ["https://iam-test.indigo-datacloud.eu/"] + Config.OIDC_AUDIENCE = None + Config.OIDC_GROUPS = ["urn:mace:egi.eu:group:demo.fedcloud.egi.eu:role=member#aai.egi.eu"] + + IM.check_oidc_token(im_auth) + + self.assertEqual(im_auth['username'], InfrastructureInfo.OPENID_USER_PREFIX + "micafer") + self.assertEqual(im_auth['password'], "https://iam-test.indigo-datacloud.eu/sub") + + Config.OIDC_GROUPS = ["urn:mace:egi.eu:group:demo.fedcloud.egi.eu:role=INVALID#aai.egi.eu"] + + with self.assertRaises(Exception) as ex: + IM.check_oidc_token(im_auth) + self.assertEqual(str(ex.exception), + "Error trying to validate OIDC auth token: Invalid InfrastructureManager" + + " credentials. User not in configured groups.") + + Config.OIDC_GROUPS = [] + def test_inf_auth_with_token(self): im_auth = {"token": (self.gen_token())} im_auth['username'] = InfrastructureInfo.OPENID_USER_PREFIX + "micafer" @@ -1488,6 +1518,47 @@ def test_translate_egi_to_ost(self, appdb): 'host': 'https://ostsite.com:5000', 'domain': 'projectid'}, res.auth_list) self.assertIn({'type': 'InfrastructureManager', 'token': 'atoken'}, res.auth_list) + def test_estimate_resources(self): + radl = """" + network publica (outbound = 'yes') + network privada + + system front ( + cpu.count>=2 and + memory.size>=4g and + net_interface.0.connection = 'publica' and + net_interface.1.connection = 'privada' and + disk.0.image.url = 'mock0://linux.for.ev.er' and + disk.0.free_size >= 20GB and + disk.0.os.name = 'linux' and + disk.1.size=100GB and + disk.1.device='hdb' and + disk.1.fstype='ext4' and + disk.1.mount_path='/mnt/disk' + ) + + system wn ( + cpu.count>=1 and + memory.size>=2g and + net_interface.0.connection = 'privada' and + disk.0.image.url = 'mock0://linux.for.ev.er' and + disk.0.free_size >= 10GB and + disk.0.os.name = 'linux' + ) + + deploy front 1 + deploy wn 1 + """ + res = IM.EstimateResouces(radl, self.getAuth([0], [], [("Dummy", 0)])) + self.assertEqual(res, { + 'cloud0': { + 'cloudType': 'Dummy', + 'cloudEndpoint': 'http://server.com:80/path', + 'compute': [{'cpuCores': 2, 'memoryInMegabytes': 4096, 'diskSizeInGigabytes': 20}, + {'cpuCores': 1, 'memoryInMegabytes': 2048, 'diskSizeInGigabytes': 10}], + 'storage': [{'sizeInGigabytes': 100}] + }}) + if __name__ == "__main__": unittest.main()