From ea9795d96a89255c036e30847ffab794cc29a15b Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 9 Jan 2023 16:16:42 +0100 Subject: [PATCH 1/7] Search VMs in site by name --- IM/InfrastructureManager.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index 45a13a5d6..2693bd984 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -35,7 +35,7 @@ from IM.VirtualMachine import VirtualMachine from radl import radl_parse -from radl.radl import Feature, RADL +from radl.radl import Feature, RADL, system from radl.radl_json import dump_radl as dump_radl_json from IM.openid.JWT import JWT @@ -386,6 +386,21 @@ def _compute_score(system_score, requested_radl): return concrete_system, score + @staticmethod + def search_vm(sel_inf, radl_sys, auth): + dist = radl_sys.getValue('disk.0.os.flavour') + version = radl_sys.getValue('disk.0.os.version') + res = [] + for c in CloudInfo.get_cloud_list(auth): + cloud_site = c.getCloudConnector(sel_inf) + for image in cloud_site.list_images(auth): + if ((dist is None or dist.lower() in image["name"].lower()) and + (version is None or version.lower() in image["name"].lower())): + new_sys = system() + new_sys.setValue("disk.0.image.url", image["uri"]) + res.append(new_sys) + return res + @staticmethod def systems_with_iis(sel_inf, radl, auth): """ @@ -411,9 +426,6 @@ def systems_with_iis(sel_inf, radl, auth): for system_id in set([d.id for d in radl.deploys if d.vm_number > 0]): s = radl.get_system_by_name(system_id) - if not s.getValue("disk.0.image.url") and len(vmrc_list + appdbis_list) == 0: - raise Exception("No correct VMRC or AppDBIS auth data provided nor image URL") - if Config.SINGLE_SITE: image_id = os.path.basename(s.getValue("disk.0.image.url")) url_prefix = Config.SINGLE_SITE_IMAGE_URL_PREFIX @@ -435,13 +447,14 @@ def systems_with_iis(sel_inf, radl, auth): vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)] appdbis_res = [s0 for appdbis in appdbis_list for s0 in appdbis.search_vm(s)] + local_res = InfrastructureManager.search_vm(sel_inf, radl, auth) # Check that now the image URL is in the RADL - if not s.getValue("disk.0.image.url") and not vmrc_res and not appdbis_res: - sel_inf.add_cont_msg("No VMI obtained from VMRC nor AppDBIS to system: " + system_id) - raise Exception("No VMI obtained from VMRC nor AppDBIS to system: " + system_id) + if not s.getValue("disk.0.image.url") and not vmrc_res and not appdbis_res and not local_res: + sel_inf.add_cont_msg("No VMI obtained from VMRC nor AppDBIS nor Sites to system: " + system_id) + raise Exception("No VMI obtained from VMRC nor AppDBIS not Sites to system: " + system_id) n = [s_without_apps.clone().applyFeatures(s0, conflict="other", missing="other") - for s0 in (vmrc_res + appdbis_res)] + for s0 in (vmrc_res + appdbis_res + local_res)] systems_with_vmrc[system_id] = n if n else [s_without_apps] return systems_with_vmrc From 247a934779425b0ec265618487ef96d9a27f0b98 Mon Sep 17 00:00:00 2001 From: micafer Date: Mon, 9 Jan 2023 16:49:37 +0100 Subject: [PATCH 2/7] Fix and add test --- IM/InfrastructureManager.py | 10 +++++++--- test/unit/test_im_logic.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index 2693bd984..dbc5c6831 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -387,16 +387,20 @@ def _compute_score(system_score, requested_radl): return concrete_system, score @staticmethod - def search_vm(sel_inf, radl_sys, auth): + def search_vm(inf, radl_sys, auth): + # If an images is already set do not search + if radl_sys.getValue("disk.0.image.url"): + return [] + dist = radl_sys.getValue('disk.0.os.flavour') version = radl_sys.getValue('disk.0.os.version') res = [] for c in CloudInfo.get_cloud_list(auth): - cloud_site = c.getCloudConnector(sel_inf) + cloud_site = c.getCloudConnector(inf) for image in cloud_site.list_images(auth): if ((dist is None or dist.lower() in image["name"].lower()) and (version is None or version.lower() in image["name"].lower())): - new_sys = system() + new_sys = system(radl_sys.name) new_sys.setValue("disk.0.image.url", image["uri"]) res.append(new_sys) return res diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py index 196762ea9..d79270bdd 100755 --- a/test/unit/test_im_logic.py +++ b/test/unit/test_im_logic.py @@ -1472,6 +1472,23 @@ def test_change_inf_auth(self): self.assertEqual(str(ex.exception), ("Invalid new infrastructure data provided: No credentials" " provided for the InfrastructureManager.")) + @patch("IM.connectors.Dummy.DummyCloudConnector") + def test_search_vm(self, dummycc): + auth = self.getAuth([0], [], [("Dummy", 0), ("Dummy", 0)]) + radl_sys = system("s0", [Feature("disk.0.os.flavour", "=", "Ubuntu"), + Feature("disk.0.os.version", "=", "20.04")]) + inf = MagicMock() + dummy = MagicMock(["list_images"]) + dummycc.return_value = dummy + dummy.list_images.side_effect = [[{"name": "ubuntu-20.04-raw", "uri": "imageuri"}], + [{"name": "ubuntu-20.04-raw", "uri": "imageuri2"}]] + res = IM.search_vm(inf, radl_sys, auth) + self.assertEqual(len(res), 2) + self.assertEqual(res[0].name, "s0") + self.assertEqual(res[0].getValue("disk.0.image.url"), "imageuri") + self.assertEqual(res[1].name, "s0") + self.assertEqual(res[1].getValue("disk.0.image.url"), "imageuri2") + if __name__ == "__main__": unittest.main() From d3f82757c8280352452c0cb415bd016565430701 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 10 Jan 2023 08:05:52 +0100 Subject: [PATCH 3/7] return first site image only --- IM/InfrastructureManager.py | 3 ++- test/unit/test_im_logic.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py index dbc5c6831..06bf8dadf 100644 --- a/IM/InfrastructureManager.py +++ b/IM/InfrastructureManager.py @@ -403,6 +403,7 @@ def search_vm(inf, radl_sys, auth): new_sys = system(radl_sys.name) new_sys.setValue("disk.0.image.url", image["uri"]) res.append(new_sys) + break return res @staticmethod @@ -451,7 +452,7 @@ def systems_with_iis(sel_inf, radl, auth): vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)] appdbis_res = [s0 for appdbis in appdbis_list for s0 in appdbis.search_vm(s)] - local_res = InfrastructureManager.search_vm(sel_inf, radl, auth) + local_res = InfrastructureManager.search_vm(sel_inf, s, auth) # Check that now the image URL is in the RADL if not s.getValue("disk.0.image.url") and not vmrc_res and not appdbis_res and not local_res: sel_inf.add_cont_msg("No VMI obtained from VMRC nor AppDBIS nor Sites to system: " + system_id) diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py index d79270bdd..cf4c53f27 100755 --- a/test/unit/test_im_logic.py +++ b/test/unit/test_im_logic.py @@ -1481,13 +1481,15 @@ def test_search_vm(self, dummycc): dummy = MagicMock(["list_images"]) dummycc.return_value = dummy dummy.list_images.side_effect = [[{"name": "ubuntu-20.04-raw", "uri": "imageuri"}], - [{"name": "ubuntu-20.04-raw", "uri": "imageuri2"}]] + [{"name": "ubuntu-22.04-raw", "uri": "imageuri2"}, + {"name": "ubuntu-20.04-raw", "uri": "imageuri3"}, + {"name": "ubuntu-20.04-raw", "uri": "imageuri4"}]] res = IM.search_vm(inf, radl_sys, auth) self.assertEqual(len(res), 2) self.assertEqual(res[0].name, "s0") self.assertEqual(res[0].getValue("disk.0.image.url"), "imageuri") self.assertEqual(res[1].name, "s0") - self.assertEqual(res[1].getValue("disk.0.image.url"), "imageuri2") + self.assertEqual(res[1].getValue("disk.0.image.url"), "imageuri3") if __name__ == "__main__": From 7c6f495dadf1e557275ee9674840adb0ee3090ca Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 10 Jan 2023 08:33:14 +0100 Subject: [PATCH 4/7] update changelog --- changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changelog b/changelog index e074b36c5..69565dda8 100644 --- a/changelog +++ b/changelog @@ -720,3 +720,9 @@ IM 1.13.0: * Add an admin user. * Add additional_dns_names field. * Enable to delete nodes using TOSCA without remove_list. + +IM 1.13.1: + * Enable to delete FaaS functions using TOSCA. + * Enable to create public router in openstack. + * Enable to define dependencies in OSCAR conn. + * Search VM Images in sites using name. From c05678b900365cb74f725114116c272e607c779d Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 10 Jan 2023 08:35:21 +0100 Subject: [PATCH 5/7] bump version 1.13.1 --- IM/__init__.py | 2 +- codemeta.json | 2 +- docker-devel/Dockerfile | 2 +- docker-py3/Dockerfile | 4 ++-- docker-py3/Dockerfile.alp | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/IM/__init__.py b/IM/__init__.py index b6b879db7..748ee858c 100644 --- a/IM/__init__.py +++ b/IM/__init__.py @@ -19,7 +19,7 @@ 'InfrastructureInfo', 'InfrastructureManager', 'recipe', 'request', 'REST', 'retry', 'ServiceRequests', 'SSH', 'SSHRetry', 'timedcall', 'UnixHTTPAdapter', 'VirtualMachine', 'VMRC', 'xmlobject'] -__version__ = '1.13.0' +__version__ = '1.13.1' __author__ = 'Miguel Caballer' diff --git a/codemeta.json b/codemeta.json index 7e9d9ae04..1de4f373a 100644 --- a/codemeta.json +++ b/codemeta.json @@ -6,7 +6,7 @@ "@type": "SoftwareSourceCode", "identifier": "im", "name": "Infrastructure Manager", - "version": "1.13.0", + "version": "1.13.1", "description": "IM is a tool that deploys complex and customized virtual infrastructures on IaaS Cloud deployments", "license": "GNU General Public License v3.0", "author": [ diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 8d407ac18..4c0f7415b 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM ubuntu:22.04 LABEL maintainer="Miguel Caballer " -LABEL version="1.13.0" +LABEL version="1.13.1" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index 10af52fcb..df189e4bb 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM ubuntu:22.04 LABEL maintainer="Miguel Caballer " -LABEL version="1.13.0" +LABEL version="1.13.1" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y python3 python3 RUN apt-get update && apt-get install --no-install-recommends -y python3-setuptools python3-pip git && \ pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-identity==1.8.0 && \ pip3 install pyOpenSSL cheroot xmltodict pymongo ansible==6.4.0&& \ - pip3 install IM==1.13.0 && \ + pip3 install IM==1.13.1 && \ apt-get purge -y python3-pip git && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ diff --git a/docker-py3/Dockerfile.alp b/docker-py3/Dockerfile.alp index 488057384..581a58676 100644 --- a/docker-py3/Dockerfile.alp +++ b/docker-py3/Dockerfile.alp @@ -1,7 +1,7 @@ # Dockerfile to create a container with the IM service FROM alpine:3.16 LABEL maintainer="Miguel Caballer " -LABEL version="1.13.0" +LABEL version="1.13.1" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" EXPOSE 8899 8800 @@ -34,7 +34,7 @@ RUN pip3 install pyOpenSSL \ RUN pip3 install ansible==6.4.0 RUN apk add --no-cache git &&\ - pip3 install IM==1.13.0 &&\ + pip3 install IM==1.13.1 &&\ apk del git # Copy a ansible.cfg with correct minimum values From 57f5cd7d4098f3c793e70ae6a6bd9a64865f58f2 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 10 Jan 2023 09:07:41 +0100 Subject: [PATCH 6/7] add remove ARG --- docker-devel/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index 4c0f7415b..1c4b3e6c5 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -1,5 +1,6 @@ # Dockerfile to create a container with the IM service FROM ubuntu:22.04 +ARG BRANCH=devel LABEL maintainer="Miguel Caballer " LABEL version="1.13.1" LABEL description="Container image to run the IM service. (http://www.grycap.upv.es/im)" From 66b9a73b34ad495857edbbe2b4dd228f09a7a1c1 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 10 Jan 2023 11:22:42 +0100 Subject: [PATCH 7/7] Fix error with cryptography 39 --- contextualization/conf-ansible.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contextualization/conf-ansible.yml b/contextualization/conf-ansible.yml index 4c951609b..43296603e 100644 --- a/contextualization/conf-ansible.yml +++ b/contextualization/conf-ansible.yml @@ -140,7 +140,7 @@ pip: name: - pyOpenSSL>20.0,<22.1.0 - - cryptography + - cryptography<39.0.0 - wheel - pyyaml - paramiko>=2.9.5