From 078f7191b56a98d35c9f27616f27259121dee36f Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 17 Dec 2019 13:20:29 +0100 Subject: [PATCH 1/5] Add search image per mp_uri --- IM/AppDB.py | 43 ++++++++++++++++++++++++++++++++++++++----- test/unit/AppDB.py | 46 ++++++++++++++++++++++++++++++---------------- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/IM/AppDB.py b/IM/AppDB.py index 08a17b0e5..5e9be270f 100644 --- a/IM/AppDB.py +++ b/IM/AppDB.py @@ -105,6 +105,7 @@ def get_image_id(site_id, image_name, vo_name): def get_image_data(str_url, stype="occi"): """ The url has this format: appdb://UPV-GRyCAP/egi.docker.ubuntu.16.04?fedcloud.egi.eu + or this one appdb://UPV-GRyCAP/83d5e854-a128-5b1f-9457-d32e10a720a6:8135 Get the Site url from the AppDB """ url = urlparse(str_url) @@ -114,17 +115,49 @@ def get_image_data(str_url, stype="occi"): site_name = url[1] image_name = url[2][1:] vo_name = url[4] + site_id = AppDB.get_site_id(site_name, stype) if not site_id: return None, None, "No site ID returned from EGI AppDB for site: %s." % site_name + site_url = AppDB.get_site_url(site_id, stype) if not site_url: return None, None, "No site URL returned from EGI AppDB for site id: %s." % site_id - image_id = AppDB.get_image_id(site_id, image_name, vo_name) - if not image_id: - return None, None, "No image ID returned from EGI AppDB for image: %s/%s/%s." % (site_id, - image_name, - vo_name) + + if not vo_name and len(image_name) >= 37 and ":" in image_name: + image_id = AppDB.get_image_id_from_uri(site_id, image_name) + if not image_id: + return None, None, "No image ID returned from EGI AppDB for image: %s/%s." % (site_id, + image_name) + else: + image_id = AppDB.get_image_id(site_id, image_name, vo_name) + if not image_id: + return None, None, "No image ID returned from EGI AppDB for image: %s/%s/%s." % (site_id, + image_name, + vo_name) + return site_url, image_id, "" return None, None, "Incorrect Protocol" + + @staticmethod + def get_image_id_from_uri(site_id, image_mp_uri): + """ + Get the image ID from the site id and image mp_uri + """ + if not image_mp_uri.startswith("http"): + image_mp_uri = "https://appdb.egi.eu/store/vo/image/%s/" % image_mp_uri + + data = AppDB.appdb_call('/rest/1.0/va_providers/%s' % site_id) + if data: + if 'provider:image' in data['appdb:appdb']['virtualization:provider']: + for image in data['appdb:appdb']['virtualization:provider']['provider:image']: + if image['@mp_uri'] == image_mp_uri: + image_basename = os.path.basename(image['@va_provider_image_id']) + parts = image_basename.split("#") + if len(parts) > 1: + return parts[1] + else: + return image_basename + + return None diff --git a/test/unit/AppDB.py b/test/unit/AppDB.py index 41ec9e723..efb25e6ed 100644 --- a/test/unit/AppDB.py +++ b/test/unit/AppDB.py @@ -51,23 +51,26 @@ def get_response(self, method, url, verify, cert=None, headers=None, data=None): http://cloud.recas.ba.infn.it:8774/v2.1/f41187320a504846b132582e172fa268 + mp_uri="https://appdb.egi.eu/store/vo/image/6c43f8b5-2e26-5e42-82d5-79bb738fa8e2:8187/" + archived="false" + vmiversion="2019.01.21" + va_provider_image_id="http://url/os_tpl#image_id" + appcname="egi.docker.ubuntu.16.04" + voname="fedcloud.egi.eu"/> + mp_uri="https://appdb.egi.eu/store/vo/image/60c0ed25-fea5-5e63-b443-034df484b502:7661/" + archived="false" + vmiversion="2019.01.21" + va_provider_image_id="http://url/os_tpl#image_id2" + appcname="egi.ubuntu.16.04" + voname="fedcloud.egi.eu"/> + mp_uri="https://appdb.egi.eu/store/vo/image/83d5e854-a128-5b1f-9457-d32e10a720a6:8135/" + archived="true" + vmiversion="2018.01.21" + va_provider_image_id="http://url/os_tpl#image_id3" + appcname="egi.ubuntu.16.04" + voname="fedcloud.egi.eu"/> """ @@ -95,10 +98,21 @@ def test_get_image_id(self, requests): res = AppDB.get_image_id("8016G0", "egi.ubuntu.16.04", "fedcloud.egi.eu") self.assertEqual(res, "image_id2") + @patch('requests.request') + def test_get_image_id_from_uri(self, requests): + requests.side_effect = self.get_response + res = AppDB.get_image_id_from_uri("8016G0", "83d5e854-a128-5b1f-9457-d32e10a720a6:8135") + self.assertEqual(res, "image_id3") + @patch('requests.request') def test_get_image_data(self, requests): requests.side_effect = self.get_response str_url = "appdb://RECAS-BARI/egi.ubuntu.16.04?fedcloud.egi.eu" - site_url, image_id, msg = AppDB.get_image_data(str_url, "openstack") + site_url, image_id, _ = AppDB.get_image_data(str_url, "openstack") + self.assertEqual(site_url, "https://cloud.recas.ba.infn.it:5000") + self.assertEqual(image_id, "image_id2") + + str_url = "appdb://RECAS-BARI/60c0ed25-fea5-5e63-b443-034df484b502:7661" + site_url, image_id, _ = AppDB.get_image_data(str_url, "openstack") self.assertEqual(site_url, "https://cloud.recas.ba.infn.it:5000") self.assertEqual(image_id, "image_id2") From a8ad2b072cc20e3b5b83fdabb201fb679e40721c Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 18 Dec 2019 08:19:24 +0100 Subject: [PATCH 2/5] Improve error message parsing --- IM/REST.py | 8 +------- IM/ServiceRequests.py | 3 ++- IM/__init__.py | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/IM/REST.py b/IM/REST.py index ce8f131e2..9bbaeeb42 100644 --- a/IM/REST.py +++ b/IM/REST.py @@ -26,6 +26,7 @@ InvaliddUserException, DisabledFunctionException) from IM.auth import Authentication from IM.config import Config +from IM import get_ex_error from radl.radl_json import parse_radl as parse_radl_json, dump_radl as dump_radl_json, featuresToSimple, radlToSimple from radl.radl import RADL, Features, Feature from IM.tosca.Tosca import Tosca @@ -304,13 +305,6 @@ def format_output(res, default_type="text/plain", field_name=None, list_field_na return info -def get_ex_error(ex): - """ - Return a secure string with the error of the exception in Py2 and Py3 - """ - return getattr(ex, 'message', ex.args[0] if len(ex.args) else repr(ex)) - - @app.route('/infrastructures/:infid', method='DELETE') def RESTDestroyInfrastructure(infid=None): try: diff --git a/IM/ServiceRequests.py b/IM/ServiceRequests.py index d85f3f9e3..9dfa1a5e4 100644 --- a/IM/ServiceRequests.py +++ b/IM/ServiceRequests.py @@ -23,6 +23,7 @@ from IM.config import Config from IM.auth import Authentication from IM import __version__ as version +from IM import get_ex_error logger = logging.getLogger('InfrastructureManager') @@ -123,7 +124,7 @@ def _execute(self): return True except Exception as ex: logger.exception(self._error_mesage) - self.set("%s" % getattr(ex, 'message', ex.args[0] if len(ex.args) else repr(ex))) + self.set(get_ex_error(ex)) return False diff --git a/IM/__init__.py b/IM/__init__.py index 815cef794..b10dc3aac 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -21,3 +21,17 @@ 'VirtualMachine', 'VMRC', 'xmlobject'] __version__ = '1.9.0' __author__ = 'Miguel Caballer' + + +def get_ex_error(ex): + """ + Return a secure string with the error of the exception in Py2 and Py3 + """ + try: + return "%s" % ex + except Exception: + pass + error = getattr(ex, 'message', None) + if not error: + error = ex.args[0] if len(ex.args) else repr(ex) + return error From cdcee39126cf83eba1f5c39d9926cf12d3feaeb1 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 18 Dec 2019 08:32:33 +0100 Subject: [PATCH 3/5] Improve error message parsing --- IM/connectors/OpenStack.py | 53 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index 89855dab6..c702ca01a 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -39,6 +39,7 @@ from IM.VirtualMachine import VirtualMachine from radl.radl import Feature from IM.AppDB import AppDB +from IM import get_ex_error class OpenStackCloudConnector(LibCloudCloudConnector): @@ -348,7 +349,7 @@ def setVolumesInfo(self, vm, node): os.path.basename(volume.extra['attachments'][0]['device'])) cont += 1 except Exception as ex: - self.log_warn("Error getting volume info: %s" % ex.args[0]) + self.log_warn("Error getting volume info: %s" % get_ex_error(ex)) def updateVMInfo(self, vm, auth_data): node = self.get_node_with_id(vm.id, auth_data) @@ -656,7 +657,7 @@ def get_router_public(self, driver, radl): if router.extra['external_gateway_info']['network_id'] in pub_nets: routers[pub_nets[router.extra['external_gateway_info']['network_id']]] = router except Exception as ex: - self.log_warn("Error listing routers: %s." % ex.args[0]) + self.log_warn("Error listing routers: %s." % get_ex_error(ex)) # try to select first the router of the net provider id if pub_net_provider_id in routers: @@ -708,7 +709,7 @@ def delete_networks(self, driver, inf): res = False msg = "Error deleting subnet %s from the router %s: %s" % (subnet_id, router.name, - ex.args[0]) + get_ex_error(ex)) self.log_info("Deleting net %s." % ost_net.name) driver.ex_delete_network(ost_net) @@ -775,7 +776,7 @@ def create_networks(self, driver, radl, inf): except Exception as ex: self.log_exception("Error creating ost network for net %s." % net_name) raise Exception("Error creating ost network for net %s: %s" % (net_name, - ex.args[0])) + get_ex_error(ex))) # now create the subnet ost_subnet_name = "im-%s-sub%s" % (inf.id, net_name) @@ -789,7 +790,7 @@ def create_networks(self, driver, radl, inf): self.log_debug("Deleting net: %s" % ost_net_name) driver.ex_delete_network(ost_net) raise Exception("Error creating ost subnet for net %s: %s" % (net_name, - ex.args[0])) + get_ex_error(ex))) if router is None: self.log_warn("No public router found.") @@ -805,7 +806,7 @@ def create_networks(self, driver, radl, inf): self.log_error("Error adding subnet to the router. Deleting net and subnet.") driver.ex_delete_subnet(ost_subnet) driver.ex_delete_network(ost_net) - raise Exception("Error adding subnet to the router: %s" % ex.args[0]) + raise Exception("Error adding subnet to the router: %s" % get_ex_error(ex)) network.setValue('provider_id', ost_net_name) except Exception as ext: @@ -1041,8 +1042,8 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm.destroy = False res.append((True, vm)) except Exception as ex: - self.log_exception("Error creating node: %s." % ex.args[0]) - res.append((False, "%s" % ex.args[0])) + self.log_exception("Error creating node: %s." % get_ex_error(ex)) + res.append((False, "%s" % get_ex_error(ex))) i += 1 @@ -1189,7 +1190,7 @@ def add_elastic_ip_from_pool(self, vm, node, fixed_ip=None, pool_name=None): node.driver.ex_attach_floating_ip_to_node(node, floating_ip) attached = True except Exception as atex: - self.log_warn("Error attaching a Floating IP to the node: %s" % atex.args[0]) + self.log_warn("Error attaching a Floating IP to the node: %s" % get_ex_error(atex)) cont += 1 if cont < retries: time.sleep(delay) @@ -1209,7 +1210,7 @@ def add_elastic_ip_from_pool(self, vm, node, fixed_ip=None, pool_name=None): except Exception as ex: self.log_exception("Error adding an Elastic/Floating IP to VM ID: %s" % vm.id) - return False, "%s" % ex.args[0] + return False, "%s" % get_ex_error(ex) def _get_security_group(self, driver, sg_name): try: @@ -1275,7 +1276,7 @@ def create_security_groups(self, driver, inf, radl): driver.ex_create_security_group_rule(sg, 'tcp', 1, 65535, source_security_group=sg) driver.ex_create_security_group_rule(sg, 'udp', 1, 65535, source_security_group=sg) except Exception as addex: - self.log_warn("Exception adding SG rules. Probably the rules exists: %s" % addex.args[0]) + self.log_warn("Exception adding SG rules. Probably the rules exists: %s" % get_ex_error(addex)) outports = network.getOutPorts() if outports: @@ -1286,7 +1287,7 @@ def create_security_groups(self, driver, inf, radl): outport.get_port_init(), outport.get_port_end(), '0.0.0.0/0') except Exception as ex: - self.log_warn("Exception adding SG rules: %s" % ex.args[0]) + self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) else: if outport.get_remote_port() != 22 or not network.isPublic(): try: @@ -1294,7 +1295,7 @@ def create_security_groups(self, driver, inf, radl): outport.get_remote_port(), outport.get_remote_port(), '0.0.0.0/0') except Exception as ex: - self.log_warn("Exception adding SG rules: %s" % ex.args[0]) + self.log_warn("Exception adding SG rules: %s" % get_ex_error(ex)) return res @@ -1315,7 +1316,7 @@ def finalize(self, vm, last, auth_data): res, msg = self.delete_elastic_ips(node, vm) except Exception as ex: res = False - msg = "%s" % ex.args[0] + msg = get_ex_error(ex) success.append(res) msgs.append(msg) @@ -1331,7 +1332,7 @@ def finalize(self, vm, last, auth_data): volume.destroy() except Exception as ex: res = False - msg = "%s" % ex.args[0] + msg = get_ex_error(ex) success.append(res) msgs.append(msg) @@ -1347,7 +1348,7 @@ def finalize(self, vm, last, auth_data): res, msg = self.delete_security_groups(driver, vm.inf) except Exception as ex: res = False - msg = "%s" % ex.args[0] + msg = get_ex_error(ex) success.append(res) msgs.append(msg) @@ -1356,7 +1357,7 @@ def finalize(self, vm, last, auth_data): res, msg = self.delete_networks(driver, vm.inf) except Exception as ex: res = False - msg = "%s" % ex.args[0] + msg = get_ex_error(ex) success.append(res) msgs.append(msg) else: @@ -1398,8 +1399,8 @@ def delete_security_groups(self, driver, inf, timeout=180, delay=10): driver.ex_delete_security_group(sg) deleted = True except Exception as ex: - self.log_warn("Error deleting the SG: %s" % ex.args[0]) - msg = "Error deleting the SG: %s" % ex.args[0] + self.log_warn("Error deleting the SG: %s" % get_ex_error(ex)) + msg = "Error deleting the SG: %s" % get_ex_error(ex) if not deleted: time.sleep(delay) @@ -1437,7 +1438,7 @@ def create_snapshot(self, vm, disk_num, image_name, auto_delete, auth_data): image = node.driver.create_image(node, image_name) except Exception as ex: self.log_exception("Error creating image.") - return False, "Error creating image: %s." % ex.args[0] + return False, "Error creating image: %s." % get_ex_error(ex) new_url = "ost://%s/%s" % (self.cloud.server, image.id) if auto_delete: vm.inf.snapshots.append(new_url) @@ -1452,13 +1453,13 @@ def delete_image(self, image_url, auth_data): image = driver.get_image(image_id) except Exception as ex: self.log_exception("Error getting image.") - return (False, "Error getting image %s: %s" % (image_id, ex.args[0])) + return (False, "Error getting image %s: %s" % (image_id, get_ex_error(ex))) try: driver.delete_image(image) return True, "" except Exception as ex: self.log_exception("Error deleting image.") - return (False, "Error deleting image.: %s" % ex.args[0]) + return (False, "Error deleting image.: %s" % get_ex_error(ex)) def reboot(self, vm, auth_data): node = self.get_node_with_id(vm.id, auth_data) @@ -1532,7 +1533,7 @@ def create_attach_volume(self, node, disk_size, disk_device, volume_name, locati volume = node.driver.create_volume(int(disk_size), volume_name, location=location) except Exception as ex: self.log_exception("Error creating volume.") - return False, None, ex.args[0] + return False, None, get_ex_error(ex) success = self.wait_volume(volume) if not success: self.log_error("Error waiting the volume ID %s." % volume.id) @@ -1543,7 +1544,7 @@ def create_attach_volume(self, node, disk_size, disk_device, volume_name, locati volume.attach(node, disk_device) except Exception as ex: self.log_exception("Error attaching volume ID %s" % volume.id) - return False, volume, ex.args[0] + return False, volume, get_ex_error(ex) # wait the volume to be attached success = self.wait_volume(volume, state='in-use') # update the volume data @@ -1656,7 +1657,7 @@ def delete_elastic_ips(self, node, vm): try: node.driver.ex_detach_floating_ip_from_node(node, floating_ip) except Exception as ex: - self.log_warn("Error detaching Floating IP: %s. %s" % (floating_ip.ip_address, ex.args[0])) + self.log_warn("Error detaching Floating IP: %s. %s" % (floating_ip.ip_address, get_ex_error(ex))) # if it is in the list do not release it if floating_ip.ip_address in no_delete_ips: self.log_debug("Do not remove Floating IP: %s" % floating_ip.ip_address) @@ -1667,4 +1668,4 @@ def delete_elastic_ips(self, node, vm): return True, "" except Exception as ex: self.log_exception("Error removing Floating IPs to VM ID: " + str(vm.id)) - return False, "Error removing Floating IPs: %s" % ex.args[0] + return False, "Error removing Floating IPs: %s" % get_ex_error(ex) From 733865decb33b83cccdb1741cb6ef5674056e9a9 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 18 Dec 2019 09:11:56 +0100 Subject: [PATCH 4/5] Fix style --- IM/connectors/OpenStack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index c702ca01a..ec1e1676d 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -1657,7 +1657,8 @@ def delete_elastic_ips(self, node, vm): try: node.driver.ex_detach_floating_ip_from_node(node, floating_ip) except Exception as ex: - self.log_warn("Error detaching Floating IP: %s. %s" % (floating_ip.ip_address, get_ex_error(ex))) + self.log_warn("Error detaching Floating IP: %s. %s" % (floating_ip.ip_address, + get_ex_error(ex))) # if it is in the list do not release it if floating_ip.ip_address in no_delete_ips: self.log_debug("Do not remove Floating IP: %s" % floating_ip.ip_address) From 2d6347a159df7dee6227b8a2eee081cc4bea1a42 Mon Sep 17 00:00:00 2001 From: micafer Date: Wed, 18 Dec 2019 09:20:45 +0100 Subject: [PATCH 5/5] Improve code --- IM/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/IM/__init__.py b/IM/__init__.py index b10dc3aac..3e0e0d0fc 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -30,8 +30,7 @@ def get_ex_error(ex): try: return "%s" % ex except Exception: - pass - error = getattr(ex, 'message', None) - if not error: - error = ex.args[0] if len(ex.args) else repr(ex) - return error + error = getattr(ex, 'message', None) + if not error: + error = ex.args[0] if len(ex.args) else repr(ex) + return error