From f206bc0f8437f9337ca33a694d65117a75582fc5 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 19 Feb 2024 13:57:02 +0100 Subject: [PATCH 01/12] Improve k8s with ingress and outputs --- IM/connectors/Kubernetes.py | 155 +++++++++++++++++++---------- IM/tosca/Tosca.py | 29 +++++- test/files/tosca_k8s.yml | 3 + test/unit/Tosca.py | 17 ++++ test/unit/connectors/Kubernetes.py | 84 ++++++++++++---- 5 files changed, 214 insertions(+), 74 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index a007c334..315d9f6b 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -37,10 +37,6 @@ class KubernetesCloudConnector(CloudConnector): type = "Kubernetes" - """ Default password to set to the root in the container""" - _apiVersions = ["v1", "v1beta3"] - """ Supported API versions""" - VM_STATE_MAP = { 'Pending': VirtualMachine.PENDING, 'Running': VirtualMachine.RUNNING, @@ -53,10 +49,6 @@ def __init__(self, cloud_info, inf): self.apiVersion = None CloudConnector.__init__(self, cloud_info, inf) - def _get_api_url(self, auth_data, namespace, path): - apiVersion = self.get_api_version(auth_data) - return "/api/" + apiVersion + "/namespaces/" + namespace + path - def create_request(self, method, url, auth_data, headers=None, body=None): auth_header = self.get_auth_header(auth_data) if auth_header: @@ -99,32 +91,6 @@ def get_auth_header(self, auth_data): return auth_header - def get_api_version(self, auth_data): - """ - Return the API version to use to connect with kubernetes API server - """ - if self.apiVersion: - return self.apiVersion - - version = self._apiVersions[0] - - try: - resp = self.create_request('GET', "/api/", auth_data) - - if resp.status_code == 200: - output = json.loads(resp.text) - for v in self._apiVersions: - if v in output["versions"]: - self.apiVersion = v - return v - - except Exception: - self.log_exception("Error connecting with Kubernetes API server") - - self.log_warn("Error getting a compatible API version. Setting the default one.") - self.log_debug("Using %s API version." % version) - return version - def concrete_system(self, radl_system, str_url, auth_data): url = urlparse(str_url) protocol = url[0] @@ -149,7 +115,7 @@ def concrete_system(self, radl_system, str_url, auth_data): def _delete_volume_claim(self, namespace, vc_name, auth_data): try: self.log_debug("Deleting PVC: %s/%s" % (namespace, vc_name)) - uri = self._get_api_url(auth_data, namespace, "/persistentvolumeclaims/" + vc_name) + uri = "/api/v1/namespaces/%s/%s/%s" % (namespace, "persistentvolumeclaims", vc_name) resp = self.create_request('DELETE', uri, auth_data) if resp.status_code == 404: @@ -176,7 +142,7 @@ def _delete_volume_claims(self, pod_data, auth_data): def _create_volume_claim(self, claim_data, auth_data): try: headers = {'Content-Type': 'application/json'} - uri = self._get_api_url(auth_data, claim_data['metadata']['namespace'], "/persistentvolumeclaims") + uri = "/api/v1/namespaces/%s/%s" % (claim_data['metadata']['namespace'], "persistentvolumeclaims") resp = self.create_request('POST', uri, auth_data, headers, claim_data) output = str(resp.text) @@ -230,7 +196,7 @@ def create_service_data(self, namespace, name, outports, auth_data, vm): service_data = self._generate_service_data(namespace, name, outports) self.log_debug("Creating Service: %s/%s" % (namespace, name)) headers = {'Content-Type': 'application/json'} - uri = self._get_api_url(auth_data, namespace, '/services') + uri = "/api/v1/namespaces/%s/%s" % (namespace, "services") svc_resp = self.create_request('POST', uri, auth_data, headers, service_data) if svc_resp.status_code != 201: self.error_messages += "Error creating service for pod %s: %s" % (name, svc_resp.text) @@ -239,12 +205,13 @@ def create_service_data(self, namespace, name, outports, auth_data, vm): # Wait a bit to assure the service has been created time.sleep(0.5) # Get Service data to get assigned nodePort - uri = self._get_api_url(auth_data, namespace, '/services/%s' % name) + uri = "/api/v1/namespaces/%s/%s/%s" % (namespace, "services", name) svc_resp = self.create_request('GET', uri, auth_data) if svc_resp.status_code == 200: for port in svc_resp.json()['spec']['ports']: # Set Out port in the RADL info of the VM - vm.setOutPort(int(port['port']), int(port['nodePort'])) + if 'nodePort' in port and port['nodePort']: + vm.setOutPort(int(port['port']), int(port['nodePort'])) except Exception: self.error_messages += "Error creating service to access pod %s" % name @@ -280,6 +247,53 @@ def _generate_service_data(self, namespace, name, outports): return service_data + def create_ingress(self, namespace, name, dns, port, auth_data): + try: + ingress_data = self._generate_ingress_data(namespace, name, dns, port) + self.log_debug("Creating Ingress: %s/%s" % (namespace, name)) + headers = {'Content-Type': 'application/json'} + uri = "/apis/networking.k8s.io/v1/namespaces/%s/ingresses" % namespace + svc_resp = self.create_request('POST', uri, auth_data, headers, ingress_data) + if svc_resp.status_code != 201: + self.error_messages += "Error creating ingress for pod %s: %s" % (name, svc_resp.text) + self.log_warn("Error creating ingress: %s" % svc_resp.text) + return False + else: + return True + except Exception: + self.error_messages += "Error creating ingress to access pod %s" % name + self.log_exception("Error creating ingress.") + return False + + def _generate_ingress_data(self, namespace, name, dns, port): + ingress_data = {'apiVersion': 'networking.k8s.io/v1', 'kind': 'Ingress'} + ingress_data['metadata'] = { + 'name': name, + 'namespace': namespace, + 'labels': {'name': name} + } + + ingress_data["spec"] = { + "rules": [ + { + "host": dns, + "http": { + "paths": [ + { + "path": "/", + "pathType": "Prefix", + "backend": { + "service": {"name": name, "port": {"number": port}} + }, + } + ] + }, + } + ] + } + + return ingress_data + @staticmethod def _get_env_variables(radl_system): env_vars = [] @@ -369,7 +383,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): # First create the namespace for the infrastructure namespace = inf.id headers = {'Content-Type': 'application/json'} - uri = self._get_api_url(auth_data, "", "") + uri = "/api/v1/namespaces/" with inf._lock: resp = self.create_request('GET', uri + namespace, auth_data, headers) if resp.status_code != 200: @@ -398,9 +412,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) vm.destroy = True inf.add_vm(vm) - (nodename, _) = vm.getRequestedName(default_hostname="pod-#N#", - default_domain=Config.DEFAULT_DOMAIN) - pod_name = nodename + pod_name = "pod-%s" % vm.im_id volumes = self._create_volumes(namespace, system, pod_name, auth_data, True) @@ -409,7 +421,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): pod_data = self._generate_pod_data(namespace, pod_name, outports, system, volumes, tags) self.log_debug("Creating POD: %s/%s" % (namespace, pod_name)) - uri = self._get_api_url(auth_data, namespace, '/pods') + uri = "/api/v1/namespaces/%s/%s" % (namespace, "pods") resp = self.create_request('POST', uri, auth_data, headers, pod_data) if resp.status_code != 201: @@ -420,6 +432,8 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): except Exception: self.log_exception("Error deleting volumes.") else: + dns_name = system.getValue("net_interface.0.dns_name") + self.create_service_data(namespace, pod_name, outports, auth_data, vm) output = json.loads(resp.text) @@ -427,6 +441,12 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm.info.systems[0].setValue('instance_id', str(vm.id)) vm.info.systems[0].setValue('instance_name', str(vm.id)) + if dns_name and outports: + port = outports[0].get_local_port() + ingress_created = self.create_ingress(namespace, pod_name, dns_name, port, auth_data) + if not ingress_created: + vm.info.systems[0].delValue("net_interface.0.dns_name") + vm.destroy = False res.append((True, vm)) @@ -441,7 +461,7 @@ def _get_pod(self, vm, auth_data): namespace = vm.inf.id pod_name = vm.id - uri = self._get_api_url(auth_data, namespace, "/pods/" + pod_name) + uri = "/api/v1/namespaces/%s/%s/%s" % (namespace, "pods", pod_name) resp = self.create_request('GET', uri, auth_data) if resp.status_code == 200: @@ -512,26 +532,33 @@ def finalize(self, vm, last, auth_data): self.log_error("Error deleting Pod %s: %s" % (vm.id, msg)) return False, "Error deleting Pod %s: %s" % (vm.id, msg) - success, msg = self._delete_service(vm, auth_data) + del_svc_ok, svc_msg = self._delete_service(vm, auth_data) + success = success and del_svc_ok + msg += svc_msg + + del_ing_ok, ing_msg = self._delete_ingress(vm, auth_data) + success = success and del_ing_ok + msg += ing_msg else: self.log_warn("No VM ID. Ignoring") success = True if last: - self._delete_namespace(vm, auth_data) + del_ns_ok, ns_msg = self._delete_namespace(vm, auth_data) + success = success and del_ns_ok + msg += ns_msg return success, msg def _delete_namespace(self, vm, auth_data): self.log_debug("Deleting Namespace: %s" % vm.inf.id) - uri = self._get_api_url(auth_data, vm.inf.id, '') + uri = "/api/v1/namespaces/%s" % vm.inf.id resp = self.create_request('DELETE', uri, auth_data) if resp.status_code == 404: self.log_warn("Trying to remove a non existing Namespace id: " + vm.inf.id) elif resp.status_code != 200: - self.log_error("Error deleting Namespace") - return False - return True + return (False, "Error deleting the Namespace: " + resp.text) + return True, "" def _delete_service(self, vm, auth_data): try: @@ -539,7 +566,7 @@ def _delete_service(self, vm, auth_data): service_name = vm.id self.log_debug("Deleting Service: %s/%s" % (namespace, service_name)) - uri = self._get_api_url(auth_data, namespace, "/services/" + service_name) + uri = "/api/v1/namespaces/%s/%s/%s" % (namespace, "services", service_name) resp = self.create_request('DELETE', uri, auth_data) if resp.status_code == 404: @@ -553,13 +580,33 @@ def _delete_service(self, vm, auth_data): self.log_exception("Error connecting with Kubernetes API server") return (False, "Error connecting with Kubernetes API server") + def _delete_ingress(self, vm, auth_data): + try: + namespace = vm.inf.id + ingress_name = vm.id + + self.log_debug("Deleting Ingress: %s/%s" % (namespace, ingress_name)) + uri = "/apis/networking.k8s.io/v1/namespaces/%s/ingresses/%s" % (namespace, ingress_name) + resp = self.create_request('DELETE', uri, auth_data) + + if resp.status_code == 404: + self.log_warn("Trying to remove a non existing Ingress id: " + ingress_name) + return (True, ingress_name) + elif resp.status_code != 200: + return (False, "Error deleting the Ingress: " + resp.text) + else: + return (True, ingress_name) + except Exception: + self.log_exception("Error connecting with Kubernetes API server") + return (False, "Error connecting with Kubernetes API server") + def _delete_pod(self, vm, auth_data): try: namespace = vm.inf.id pod_name = vm.id self.log_debug("Deleting POD: %s/%s" % (namespace, pod_name)) - uri = self._get_api_url(auth_data, namespace, "/pods/" + pod_name) + uri = "/api/v1/namespaces/%s/%s/%s" % (namespace, "pods", pod_name) resp = self.create_request('DELETE', uri, auth_data) if resp.status_code == 404: @@ -612,7 +659,7 @@ def alterVM(self, vm, radl, auth_data): pod_name = vm.id headers = {'Content-Type': 'application/json-patch+json'} - uri = self._get_api_url(auth_data, namespace, "/pods/" + pod_name) + uri = "/api/v1/namespaces/%s/%s/%s" % (namespace, "pods", pod_name) resp = self.create_request('PATCH', uri, auth_data, headers, pod_data) if resp.status_code != 200: diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 0c55889a..7eb1fb4b 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -1206,7 +1206,8 @@ def _get_attribute_result(self, func, node, inf_info): root_type = Tosca._get_root_parent_type(node).type host_node = self._find_host_node(node, self.tosca.nodetemplates) - if root_type == "tosca.nodes.aisprint.FaaS.Function" and host_node is None: + if (host_node is None and (root_type == "tosca.nodes.aisprint.FaaS.Function" or + node.type == "tosca.nodes.Container.Application.Docker")): # in case of FaaS functions without host, the node is the host host_node = node @@ -1352,6 +1353,32 @@ def _get_attribute_result(self, func, node, inf_info): Tosca.logger.warning("Attribute credential only supported in tosca.nodes.aisprint.FaaS.Function") return None + elif attribute_name == "endpoints": + if node.type == "tosca.nodes.Container.Application.Docker": + res = [] + dmsname = vm.info.systems[0].getValue("net_interface.0.dns_name") + pub_net = vm.getConnectedNet(public=True) + priv_net = vm.getConnectedNet(public=False) + if pub_net: + url = "http://%s" % vm.getPublicIP() + if pub_net.getOutPorts(): + for outport in pub_net.getOutPorts(): + # if DNS name is set, the endpoint of the first public port is the DNS name + if dmsname and not res: + res.append("https://%s" % dmsname) + else: + res.append(url + ":%s" % outport.get_remote_port()) + if priv_net: + url = "http://%s" % vm.getPrivateIP() + if priv_net.getOutPorts(): + for outport in priv_net.getOutPorts(): + res.append(url + ":%s" % outport.get_remote_port()) + + if index is not None: + res = res[index] + return res + Tosca.logger.warning("Attribute credential only supported in tosca.nodes.Container.Application.Docker") + return None else: Tosca.logger.warning("Attribute %s not supported." % attribute_name) return None diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 48f57227..242220a6 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -56,3 +56,6 @@ topology_template: # Set the PV name in this field # volume_id: "PV name" + outputs: + mysql_service_url: + value: { get_attribute: [ mysql_container, endpoints, 0 ] } diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 3160362f..347ba192 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -447,6 +447,23 @@ def test_tosca_k8s(self): conf = radl.get_configure_by_name('mysql_container') self.assertEqual(conf.recipes, None) + def test_tosca_k8s_get_attribute(self): + """Test TOSCA K8s get_attributes function""" + tosca_data = read_file_as_string('../files/tosca_k8s.yml') + tosca = Tosca(tosca_data) + _, radl = tosca.to_radl() + radl1 = radl.clone() + radl1.systems = [radl.get_system_by_name('mysql_container')] + inf = InfrastructureInfo() + radl1.systems[0].setValue("net_interface.0.ip", "8.8.8.8") + + cloud_info = MagicMock() + vm = VirtualMachine(inf, "1", cloud_info, radl1, radl1, None) + vm.requested_radl = radl1 + inf.vm_list = [vm] + outputs = tosca.get_outputs(inf) + self.assertEqual(outputs, {'mysql_service_url': 'http://8.8.8.8:33306'}) + if __name__ == "__main__": unittest.main() diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index a6d402a9..62c562d6 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -98,6 +98,8 @@ def get_response(self, method, url, verify, headers, data): resp.status_code = 201 elif url.endswith("/persistentvolumeclaims"): resp.status_code = 201 + elif url.endswith("/apis/networking.k8s.io/v1/namespaces/infid/ingresses"): + resp.status_code = 201 elif method == "DELETE": if url.endswith("/pods/1"): resp.status_code = 200 @@ -107,12 +109,17 @@ def get_response(self, method, url, verify, headers, data): resp.status_code = 200 elif "persistentvolumeclaims" in url: resp.status_code = 200 + elif "ingresses" in url: + resp.status_code = 200 elif method == "PATCH": if url.endswith("/pods/1"): resp.status_code = 200 return resp + def add_vm(self, vm): + vm.im_id = 0 + @patch('requests.request') @patch('IM.InfrastructureList.InfrastructureList.save_data') def test_20_launch(self, save_data, requests): @@ -122,7 +129,7 @@ def test_20_launch(self, save_data, requests): cpu.count>=1 and memory.size>=512m and net_interface.0.connection = 'net' and - net_interface.0.dns_name = 'test' and + net_interface.0.dns_name = 'ingress.domain.com' and environment.variables = 'var=some_val' and instance_tags = 'key=_inva:lid_' and disk.0.os.name = 'linux' and @@ -141,6 +148,7 @@ def test_20_launch(self, save_data, requests): inf = MagicMock(["id", "_lock", "add_vm"]) inf.id = "infid" + inf.add_vm.side_effect = self.add_vm res = kube_cloud.launch(inf, radl, radl, 1, auth) success, _ = res[0] self.assertTrue(success, msg="ERROR: launching a VM.") @@ -148,28 +156,28 @@ def test_20_launch(self, save_data, requests): exp_pvc = { "apiVersion": "v1", "kind": "PersistentVolumeClaim", - "metadata": {"name": "test-1", "namespace": "infid"}, + "metadata": {"name": "pod-0-1", "namespace": "infid"}, "spec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": 10737418240}}, }, } - self.assertEqual(requests.call_args_list[2][0][1], + self.assertEqual(requests.call_args_list[1][0][1], 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims') - self.assertEqual(json.loads(requests.call_args_list[2][1]['data']), exp_pvc) + self.assertEqual(json.loads(requests.call_args_list[1][1]['data']), exp_pvc) exp_pod = { "apiVersion": "v1", "kind": "Pod", "metadata": { - "name": "test", + "name": "pod-0", "namespace": "infid", - "labels": {"name": "test", "IM_INFRA_ID": "infid", "key": "invalid_"}, + "labels": {"name": "pod-0", "IM_INFRA_ID": "infid", "key": "invalid_"}, }, "spec": { "containers": [ { - "name": "test", + "name": "pod-0", "image": "someimage", "imagePullPolicy": "Always", "ports": [{"containerPort": 8080, "protocol": "TCP"}], @@ -178,25 +186,25 @@ def test_20_launch(self, save_data, requests): "requests": {"cpu": "1", "memory": "536870912"}, }, "env": [{"name": "var", "value": "some_val"}], - "volumeMounts": [{"name": "test-1", "mountPath": "/mnt"}], + "volumeMounts": [{"name": "pod-0-1", "mountPath": "/mnt"}], } ], "restartPolicy": "OnFailure", "volumes": [ - {"name": "test-1", "persistentVolumeClaim": {"claimName": "test-1"}} + {"name": "pod-0-1", "persistentVolumeClaim": {"claimName": "pod-0-1"}} ], }, } - self.assertEqual(requests.call_args_list[3][0][1], 'http://server.com:8080/api/v1/namespaces/infid/pods') - self.assertEqual(json.loads(requests.call_args_list[3][1]['data']), exp_pod) + self.assertEqual(requests.call_args_list[2][0][1], 'http://server.com:8080/api/v1/namespaces/infid/pods') + self.assertEqual(json.loads(requests.call_args_list[2][1]['data']), exp_pod) exp_svc = { "apiVersion": "v1", "kind": "Service", "metadata": { - "name": "test", + "name": "pod-0", "namespace": "infid", - "labels": {"name": "test"}, + "labels": {"name": "pod-0"}, }, "spec": { "type": "NodePort", @@ -209,11 +217,46 @@ def test_20_launch(self, save_data, requests): "nodePort": 38080, } ], - "selector": {"name": "test"}, + "selector": {"name": "pod-0"}, + }, + } + self.assertEqual(requests.call_args_list[3][0][1], 'http://server.com:8080/api/v1/namespaces/infid/services') + self.assertEqual(json.loads(requests.call_args_list[3][1]['data']), exp_svc) + + exp_ing = { + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + "metadata": { + "labels": {"name": "pod-0"}, + "name": "pod-0", + "namespace": "infid", + }, + "spec": { + "rules": [ + { + "host": "ingress.domain.com", + "http": { + "paths": [ + { + "backend": { + "service": { + "name": "pod-0", + "port": {"number": 8080}, + } + }, + "path": "/", + "pathType": "Prefix", + } + ] + }, + } + ] }, } - self.assertEqual(requests.call_args_list[4][0][1], 'http://server.com:8080/api/v1/namespaces/infid/services') - self.assertEqual(json.loads(requests.call_args_list[4][1]['data']), exp_svc) + + self.assertEqual(requests.call_args_list[5][0][1], + 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/infid/ingresses') + self.assertEqual(json.loads(requests.call_args_list[5][1]['data']), exp_ing) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @@ -296,15 +339,18 @@ def test_60_finalize(self, requests): success, _ = kube_cloud.finalize(vm, True, auth) - self.assertEqual(requests.call_args_list[2][0], + self.assertEqual(requests.call_args_list[1][0], ('DELETE', 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims/cname')) - self.assertEqual(requests.call_args_list[3][0], + self.assertEqual(requests.call_args_list[2][0], ('DELETE', 'http://server.com:8080/api/v1/namespaces/infid/pods/1')) - self.assertEqual(requests.call_args_list[4][0], + self.assertEqual(requests.call_args_list[3][0], ('DELETE', 'http://server.com:8080/api/v1/namespaces/infid/services/1')) + self.assertEqual(requests.call_args_list[4][0], + ('DELETE', + 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/infid/ingresses/1')) self.assertEqual(requests.call_args_list[5][0], ('DELETE', 'http://server.com:8080/api/v1/namespaces/infid')) From fa98ba86f0203094ada83153c126916724c956a2 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Mon, 19 Feb 2024 13:57:20 +0100 Subject: [PATCH 02/12] Improve k8s with ingress and outputs --- IM/VirtualMachine.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 24bb3920..1a7883b1 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -267,6 +267,16 @@ def getPrivateIP(self): """ return self.info.getPrivateIP() + def getConnectedNet(self, public=False): + """ + Return the first public/private net connected to this VM if available + """ + for netid in self.info.systems[0].getNetworkIDs(): + net = self.info.get_network_by_id(netid) + if net.isPublic() == public: + return net + return None + def getNumNetworkIfaces(self): """ Get the number of net interfaces of this VM From 139984516028a12c4d21dc8c25bfa1d16e3abc97 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 02:05:50 +0100 Subject: [PATCH 03/12] Improve code to enable multi container app --- IM/connectors/Kubernetes.py | 100 ++++++++++++++++------------- IM/tosca/Tosca.py | 50 +++++++++++++-- requirements-tests.txt | 1 + setup.py | 2 +- test/files/tosca_k8s.yml | 40 ++++++++++-- test/unit/Tosca.py | 29 +++++++-- test/unit/connectors/Kubernetes.py | 70 +++++++++++--------- 7 files changed, 200 insertions(+), 92 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 315d9f6b..229c3299 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -367,6 +367,13 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes, tags): return pod_data + @staticmethod + def _get_namespace(inf): + namespace = inf.id + if inf.description and inf.description.getValue('namespace'): + namespace = inf.description.getValue('namespace') + return namespace + def launch(self, inf, radl, requested_radl, num_vm, auth_data): system = radl.systems[0] @@ -381,7 +388,8 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res = [] # First create the namespace for the infrastructure - namespace = inf.id + namespace = self._get_namespace(inf) + headers = {'Content-Type': 'application/json'} uri = "/api/v1/namespaces/" with inf._lock: @@ -404,61 +412,60 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append((False, "Error creating the Namespace")) return res - i = 0 - while i < num_vm: - try: - i += 1 + if num_vm != 1: + self.log_warn("Num VM is not 1. Ignoring.") - vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) - vm.destroy = True - inf.add_vm(vm) - pod_name = "pod-%s" % vm.im_id + try: + vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) + vm.destroy = True + inf.add_vm(vm) + pod_name = system.name - volumes = self._create_volumes(namespace, system, pod_name, auth_data, True) + volumes = self._create_volumes(namespace, system, pod_name, auth_data, True) - tags = self.get_instance_tags(system, auth_data, inf) + tags = self.get_instance_tags(system, auth_data, inf) - pod_data = self._generate_pod_data(namespace, pod_name, outports, system, volumes, tags) + pod_data = self._generate_pod_data(namespace, pod_name, outports, system, volumes, tags) - self.log_debug("Creating POD: %s/%s" % (namespace, pod_name)) - uri = "/api/v1/namespaces/%s/%s" % (namespace, "pods") - resp = self.create_request('POST', uri, auth_data, headers, pod_data) + self.log_debug("Creating POD: %s/%s" % (namespace, pod_name)) + uri = "/api/v1/namespaces/%s/%s" % (namespace, "pods") + resp = self.create_request('POST', uri, auth_data, headers, pod_data) - if resp.status_code != 201: - self.log_error("Error creating the Container: " + resp.text) - res.append((False, "Error creating the Container: " + resp.text)) - try: - self._delete_volume_claims(pod_data, auth_data) - except Exception: - self.log_exception("Error deleting volumes.") - else: - dns_name = system.getValue("net_interface.0.dns_name") + if resp.status_code != 201: + self.log_error("Error creating the Container: " + resp.text) + res.append((False, "Error creating the Container: " + resp.text)) + try: + self._delete_volume_claims(pod_data, auth_data) + except Exception: + self.log_exception("Error deleting volumes.") + else: + dns_name = system.getValue("net_interface.0.dns_name") - self.create_service_data(namespace, pod_name, outports, auth_data, vm) + self.create_service_data(namespace, pod_name, outports, auth_data, vm) - output = json.loads(resp.text) - vm.id = output["metadata"]["name"] - vm.info.systems[0].setValue('instance_id', str(vm.id)) - vm.info.systems[0].setValue('instance_name', str(vm.id)) + output = json.loads(resp.text) + vm.id = output["metadata"]["name"] + vm.info.systems[0].setValue('instance_id', str(vm.id)) + vm.info.systems[0].setValue('instance_name', str(vm.id)) - if dns_name and outports: - port = outports[0].get_local_port() - ingress_created = self.create_ingress(namespace, pod_name, dns_name, port, auth_data) - if not ingress_created: - vm.info.systems[0].delValue("net_interface.0.dns_name") + if dns_name and outports: + port = outports[0].get_local_port() + ingress_created = self.create_ingress(namespace, pod_name, dns_name, port, auth_data) + if not ingress_created: + vm.info.systems[0].delValue("net_interface.0.dns_name") - vm.destroy = False - res.append((True, vm)) + vm.destroy = False + res.append((True, vm)) - except Exception as ex: - self.log_exception("Error connecting with Kubernetes API server") - res.append((False, "ERROR: " + str(ex))) + except Exception as ex: + self.log_exception("Error connecting with Kubernetes API server") + res.append((False, "ERROR: " + str(ex))) return res def _get_pod(self, vm, auth_data): try: - namespace = vm.inf.id + namespace = self._get_namespace(vm.inf) pod_name = vm.id uri = "/api/v1/namespaces/%s/%s/%s" % (namespace, "pods", pod_name) @@ -551,8 +558,9 @@ def finalize(self, vm, last, auth_data): return success, msg def _delete_namespace(self, vm, auth_data): - self.log_debug("Deleting Namespace: %s" % vm.inf.id) - uri = "/api/v1/namespaces/%s" % vm.inf.id + namespace = self._get_namespace(vm.inf) + self.log_debug("Deleting Namespace: %s" % namespace) + uri = "/api/v1/namespaces/%s" % namespace resp = self.create_request('DELETE', uri, auth_data) if resp.status_code == 404: self.log_warn("Trying to remove a non existing Namespace id: " + vm.inf.id) @@ -562,7 +570,7 @@ def _delete_namespace(self, vm, auth_data): def _delete_service(self, vm, auth_data): try: - namespace = vm.inf.id + namespace = self._get_namespace(vm.inf) service_name = vm.id self.log_debug("Deleting Service: %s/%s" % (namespace, service_name)) @@ -582,7 +590,7 @@ def _delete_service(self, vm, auth_data): def _delete_ingress(self, vm, auth_data): try: - namespace = vm.inf.id + namespace = self._get_namespace(vm.inf) ingress_name = vm.id self.log_debug("Deleting Ingress: %s/%s" % (namespace, ingress_name)) @@ -602,7 +610,7 @@ def _delete_ingress(self, vm, auth_data): def _delete_pod(self, vm, auth_data): try: - namespace = vm.inf.id + namespace = self._get_namespace(vm.inf) pod_name = vm.id self.log_debug("Deleting POD: %s/%s" % (namespace, pod_name)) @@ -655,7 +663,7 @@ def alterVM(self, vm, radl, auth_data): return (True, vm) # Create the container - namespace = vm.inf.id + namespace = self._get_namespace(vm.inf) pod_name = vm.id headers = {'Content-Type': 'application/json-patch+json'} diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 7eb1fb4b..8704cd56 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -284,9 +284,11 @@ def to_radl(self, inf_info=None): self._check_private_networks(radl, inf_info) infra_name = self.tosca.tpl.get('metadata', {}).get('infra_name') - if infra_name: + namespace = self.tosca.tpl.get('metadata', {}).get('namespace') + if infra_name or namespace: radl.description = description('desc') radl.description.setValue('name', infra_name) + radl.description.setValue('namespace', namespace) return all_removal_list, self._complete_radl_networks(radl) @@ -1360,19 +1362,20 @@ def _get_attribute_result(self, func, node, inf_info): pub_net = vm.getConnectedNet(public=True) priv_net = vm.getConnectedNet(public=False) if pub_net: - url = "http://%s" % vm.getPublicIP() + url = "%s" % vm.getPublicIP() if pub_net.getOutPorts(): for outport in pub_net.getOutPorts(): # if DNS name is set, the endpoint of the first public port is the DNS name if dmsname and not res: - res.append("https://%s" % dmsname) + res.append("%s" % dmsname) else: res.append(url + ":%s" % outport.get_remote_port()) if priv_net: - url = "http://%s" % vm.getPrivateIP() + # set the internal DNS name + url = "%s.%s" % (vm.info.systems[0].name, inf_info.id) if priv_net.getOutPorts(): for outport in priv_net.getOutPorts(): - res.append(url + ":%s" % outport.get_remote_port()) + res.append(url + ":%s" % outport.get_local_port()) if index is not None: res = res[index] @@ -1440,6 +1443,40 @@ def _get_attribute_result(self, func, node, inf_info): Tosca.logger.warning("Attribute endpoint only supported in tosca.nodes.aisprint.FaaS.Function") return None + elif attribute_name == "endpoints": + if node.type == "tosca.nodes.Container.Application.Docker": + k8s_sys, nets = self._gen_k8s_system(node, self.tosca.nodetemplates) + if nets: + net = nets[0] + else: + if index is not None: + return None + else: + return [] + + res = [] + if net.isPublic(): + dmsname = k8s_sys.getValue("net_interface.0.dns_name") + if net.getOutPorts(): + for outport in net.getOutPorts(): + # if DNS name is set, the endpoint of the first public port is the DNS name + if dmsname and not res: + res.append("%s" % dmsname) + else: + res.append("{{ hostvars[groups['%s'][0]]['IM_NODE_PUBLIC_IP'] }}:%s" % + outport.get_remote_port()) + else: + # set the internal DNS name + url = node.name + if net.getOutPorts(): + for outport in net.getOutPorts(): + res.append(url + ":%s" % outport.get_local_port()) + + if index is not None: + res = res[index] + return res + Tosca.logger.warning("Attribute credential only supported in tosca.nodes.Container.Application.Docker") + return None else: Tosca.logger.warning("Attribute %s not supported." % attribute_name) return None @@ -2237,10 +2274,13 @@ def _gen_k8s_system(self, node, nodetemplates): pub.setValue("outbound", "yes") pub.setValue("outports", self._format_outports(value)) nets.append(pub) + res.setValue("net_interface.0.connection", "%s_pub" % node.name) elif prop.name == 'expose_ports': # Asume that publish_ports must be published as ClusterIP priv = network("%s_priv" % node.name) + priv.setValue("outbound", "no") priv.setValue("outports", self._format_outports(value)) nets.append(priv) + res.setValue("net_interface.0.connection", "%s_priv" % node.name) return res, nets diff --git a/requirements-tests.txt b/requirements-tests.txt index 046c9c96..0bce9366 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -35,3 +35,4 @@ scar mock coverage requests-cache +packaging diff --git a/setup.py b/setup.py index 5a4e24ff..9727677c 100644 --- a/setup.py +++ b/setup.py @@ -70,5 +70,5 @@ install_requires=["ansible >=2.4", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot", "boto >= 2.29", "apache-libcloud >= 3.2.0", "RADL >= 1.1.5", "bottle", "netaddr", "requests >= 2.19", "scp", "tosca-parser", 'defusedxml', 'urllib3>=1.23', 'hvac', - 'psutil', 'scar', 'requests-cache >= 1.0.0'] + 'psutil', 'scar', 'requests-cache >= 1.0.0', 'packaging'] ) diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 242220a6..9020852a 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -5,6 +5,10 @@ imports: description: TOSCA test for K8s +metadata: + namespace: some_namespace + infra_name: some_infra + repositories: docker_hub: docker.io @@ -20,6 +24,32 @@ topology_template: node_templates: + im_container: + type: tosca.nodes.Container.Application.Docker + properties: + environment: + IM_DATA_DB: { concat: [ "mysql://root:my-secret@", { get_attribute: [ mysql_container, endpoints, 0 ] } ] } + requirements: + - host: im_runtime + artifacts: + my_image: + file: grycap/im + type: tosca.artifacts.Deployment.Image.Container.Docker + repository: docker_hub + + # The properties of the runtime to host the container + im_runtime: + type: tosca.nodes.Container.Runtime.Docker + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 2 GB + publish_ports: + - protocol: tcp + target: 8800 + source: 30880 + # The MYSQL container based on official MySQL image in Docker hub mysql_container: type: tosca.nodes.Container.Application.Docker @@ -42,10 +72,10 @@ topology_template: properties: num_cpus: 1 mem_size: 2 GB - publish_ports: + expose_ports: - protocol: tcp target: 3306 - source: 33306 + source: 3306 volumes: - "some_vol:/some/path" @@ -57,5 +87,7 @@ topology_template: # volume_id: "PV name" outputs: - mysql_service_url: - value: { get_attribute: [ mysql_container, endpoints, 0 ] } + im_service_endpoint: + value: { get_attribute: [ im_container, endpoints, 0 ] } + mysql_service_endpoint: + value: { get_attribute: [ mysql_container, endpoints, 0 ] } \ No newline at end of file diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 347ba192..63854b4d 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -434,6 +434,7 @@ def test_tosca_k8s(self): radl = parse_radl(str(radl)) radl.check() + self.assertEqual(radl.description.getValue("namespace"), "some_namespace") node = radl.get_system_by_name('mysql_container') self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/mysql:5.7") self.assertEqual(node.getValue("cpu.count"), 1.0) @@ -441,28 +442,44 @@ def test_tosca_k8s(self): self.assertEqual(node.getValue("disk.1.size"), 10000000000) self.assertEqual(node.getValue("disk.1.mount_path"), '/some/path') self.assertEqual(node.getValue("environment.variables"), 'MYSQL_ROOT_PASSWORD=my-secret') - net = radl.get_network_by_id('mysql_container_pub') - self.assertEqual(net.getValue("outports"), '33306/tcp-3306/tcp') - self.assertEqual(net.getValue("outbound"), 'yes') + net = radl.get_network_by_id('mysql_container_priv') + self.assertEqual(net.getValue("outports"), '3306/tcp-3306/tcp') + self.assertEqual(net.getValue("outbound"), 'no') conf = radl.get_configure_by_name('mysql_container') self.assertEqual(conf.recipes, None) + node = radl.get_system_by_name('im_container') + self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/grycap/im") + net = radl.get_network_by_id('im_container_pub') + self.assertEqual(net.getValue("outports"), '30880/tcp-8800/tcp') + self.assertEqual(net.getValue("outbound"), 'yes') + self.assertEqual(node.getValue("environment.variables"), + 'IM_DATA_DB=mysql://root:my-secret@mysql_container:3306') + conf = radl.get_configure_by_name('im_container') + self.assertEqual(conf.recipes, None) + def test_tosca_k8s_get_attribute(self): """Test TOSCA K8s get_attributes function""" tosca_data = read_file_as_string('../files/tosca_k8s.yml') tosca = Tosca(tosca_data) _, radl = tosca.to_radl() radl1 = radl.clone() - radl1.systems = [radl.get_system_by_name('mysql_container')] + radl1.systems = [radl.get_system_by_name('im_container')] inf = InfrastructureInfo() radl1.systems[0].setValue("net_interface.0.ip", "8.8.8.8") + radl2 = radl.clone() + radl2.systems = [radl.get_system_by_name('mysql_container')] + cloud_info = MagicMock() vm = VirtualMachine(inf, "1", cloud_info, radl1, radl1, None) + vm2 = VirtualMachine(inf, "2", cloud_info, radl2, radl2, None) vm.requested_radl = radl1 - inf.vm_list = [vm] + vm2.requested_radl = radl2 + inf.vm_list = [vm, vm2] outputs = tosca.get_outputs(inf) - self.assertEqual(outputs, {'mysql_service_url': 'http://8.8.8.8:33306'}) + self.assertEqual(outputs, {'im_service_endpoint': '8.8.8.8:30880', + 'mysql_service_endpoint': 'mysql_container.%s:3306' % inf.id}) if __name__ == "__main__": diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 62c562d6..9fbe938c 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -82,30 +82,30 @@ def get_response(self, method, url, verify, headers, data): resp.text = '{"versions": "v1"}' elif url.endswith("/pods/1"): resp.status_code = 200 - resp.text = ('{"metadata": {"namespace":"infid", "name": "name"}, "status": ' + resp.text = ('{"metadata": {"namespace":"some_namespace", "name": "name"}, "status": ' '{"phase":"Running", "hostIP": "158.42.1.1", "podIP": "10.0.0.1"}, ' '"spec": {"containers": [{"image": "image:1.0"}], ' '"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}') - if url == "/api/v1/namespaces/infid": + if url == "/api/v1/namespaces/some_namespace": resp.status_code = 200 elif method == "POST": if url.endswith("/pods"): resp.status_code = 201 - resp.text = '{"metadata": {"namespace":"infid", "name": "name"}}' + resp.text = '{"metadata": {"namespace":"some_namespace", "name": "name"}}' elif url.endswith("/services"): resp.status_code = 201 elif url.endswith("/namespaces/"): resp.status_code = 201 elif url.endswith("/persistentvolumeclaims"): resp.status_code = 201 - elif url.endswith("/apis/networking.k8s.io/v1/namespaces/infid/ingresses"): + elif url.endswith("/apis/networking.k8s.io/v1/namespaces/some_namespace/ingresses"): resp.status_code = 201 elif method == "DELETE": if url.endswith("/pods/1"): resp.status_code = 200 elif url.endswith("/services/1"): resp.status_code = 200 - elif url.endswith("/namespaces/infid"): + elif url.endswith("/namespaces/some_namespace"): resp.status_code = 200 elif "persistentvolumeclaims" in url: resp.status_code = 200 @@ -124,6 +124,10 @@ def add_vm(self, vm): @patch('IM.InfrastructureList.InfrastructureList.save_data') def test_20_launch(self, save_data, requests): radl_data = """ + description desc ( + name = 'Infrastructure Name' and + namespace = 'some_namespace' + ) network net (outbound = 'yes' and outports = '38080-8080') system test ( cpu.count>=1 and @@ -146,8 +150,10 @@ def test_20_launch(self, save_data, requests): requests.side_effect = self.get_response - inf = MagicMock(["id", "_lock", "add_vm"]) + inf = MagicMock(["id", "_lock", "add_vm", "description"]) inf.id = "infid" + inf.description = MagicMock(["getValue"]) + inf.description.getValue.return_value = "some_namespace" inf.add_vm.side_effect = self.add_vm res = kube_cloud.launch(inf, radl, radl, 1, auth) success, _ = res[0] @@ -156,28 +162,28 @@ def test_20_launch(self, save_data, requests): exp_pvc = { "apiVersion": "v1", "kind": "PersistentVolumeClaim", - "metadata": {"name": "pod-0-1", "namespace": "infid"}, + "metadata": {"name": "test-1", "namespace": "some_namespace"}, "spec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": 10737418240}}, }, } self.assertEqual(requests.call_args_list[1][0][1], - 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims') + 'http://server.com:8080/api/v1/namespaces/some_namespace/persistentvolumeclaims') self.assertEqual(json.loads(requests.call_args_list[1][1]['data']), exp_pvc) exp_pod = { "apiVersion": "v1", "kind": "Pod", "metadata": { - "name": "pod-0", - "namespace": "infid", - "labels": {"name": "pod-0", "IM_INFRA_ID": "infid", "key": "invalid_"}, + "name": "test", + "namespace": "some_namespace", + "labels": {"name": "test", "IM_INFRA_ID": "infid", "key": "invalid_"}, }, "spec": { "containers": [ { - "name": "pod-0", + "name": "test", "image": "someimage", "imagePullPolicy": "Always", "ports": [{"containerPort": 8080, "protocol": "TCP"}], @@ -186,25 +192,26 @@ def test_20_launch(self, save_data, requests): "requests": {"cpu": "1", "memory": "536870912"}, }, "env": [{"name": "var", "value": "some_val"}], - "volumeMounts": [{"name": "pod-0-1", "mountPath": "/mnt"}], + "volumeMounts": [{"name": "test-1", "mountPath": "/mnt"}], } ], "restartPolicy": "OnFailure", "volumes": [ - {"name": "pod-0-1", "persistentVolumeClaim": {"claimName": "pod-0-1"}} + {"name": "test-1", "persistentVolumeClaim": {"claimName": "test-1"}} ], }, } - self.assertEqual(requests.call_args_list[2][0][1], 'http://server.com:8080/api/v1/namespaces/infid/pods') + self.assertEqual(requests.call_args_list[2][0][1], + 'http://server.com:8080/api/v1/namespaces/some_namespace/pods') self.assertEqual(json.loads(requests.call_args_list[2][1]['data']), exp_pod) exp_svc = { "apiVersion": "v1", "kind": "Service", "metadata": { - "name": "pod-0", - "namespace": "infid", - "labels": {"name": "pod-0"}, + "name": "test", + "namespace": "some_namespace", + "labels": {"name": "test"}, }, "spec": { "type": "NodePort", @@ -217,19 +224,20 @@ def test_20_launch(self, save_data, requests): "nodePort": 38080, } ], - "selector": {"name": "pod-0"}, + "selector": {"name": "test"}, }, } - self.assertEqual(requests.call_args_list[3][0][1], 'http://server.com:8080/api/v1/namespaces/infid/services') + self.assertEqual(requests.call_args_list[3][0][1], + 'http://server.com:8080/api/v1/namespaces/some_namespace/services') self.assertEqual(json.loads(requests.call_args_list[3][1]['data']), exp_svc) exp_ing = { "apiVersion": "networking.k8s.io/v1", "kind": "Ingress", "metadata": { - "labels": {"name": "pod-0"}, - "name": "pod-0", - "namespace": "infid", + "labels": {"name": "test"}, + "name": "test", + "namespace": "some_namespace", }, "spec": { "rules": [ @@ -240,7 +248,7 @@ def test_20_launch(self, save_data, requests): { "backend": { "service": { - "name": "pod-0", + "name": "test", "port": {"number": 8080}, } }, @@ -255,7 +263,7 @@ def test_20_launch(self, save_data, requests): } self.assertEqual(requests.call_args_list[5][0][1], - 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/infid/ingresses') + 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/some_namespace/ingresses') self.assertEqual(json.loads(requests.call_args_list[5][1]['data']), exp_ing) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @@ -333,6 +341,8 @@ def test_60_finalize(self, requests): inf = MagicMock() inf.id = "infid" + inf.description = MagicMock(["getValue"]) + inf.description.getValue.return_value = "some_namespace" vm = VirtualMachine(inf, "1", kube_cloud.cloud, "", "", kube_cloud, 1) requests.side_effect = self.get_response @@ -341,19 +351,19 @@ def test_60_finalize(self, requests): self.assertEqual(requests.call_args_list[1][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid/persistentvolumeclaims/cname')) + 'http://server.com:8080/api/v1/namespaces/some_namespace/persistentvolumeclaims/cname')) self.assertEqual(requests.call_args_list[2][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid/pods/1')) + 'http://server.com:8080/api/v1/namespaces/some_namespace/pods/1')) self.assertEqual(requests.call_args_list[3][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid/services/1')) + 'http://server.com:8080/api/v1/namespaces/some_namespace/services/1')) self.assertEqual(requests.call_args_list[4][0], ('DELETE', - 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/infid/ingresses/1')) + 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/some_namespace/ingresses/1')) self.assertEqual(requests.call_args_list[5][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/infid')) + 'http://server.com:8080/api/v1/namespaces/some_namespace')) self.assertTrue(success, msg="ERROR: finalizing VM info.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) From 68fb7852461961badc0505da552ec122fbcbaff6 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 02:09:33 +0100 Subject: [PATCH 04/12] Improve code to enable multi container app --- test/files/tosca_k8s.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 9020852a..691b64be 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -21,6 +21,11 @@ topology_template: type: string description: The image to be used in the container default: "mysql:5.7" + + mysql_root_password: + type: string + description: The root password for the MySQL container + default: "my-secret" node_templates: @@ -28,7 +33,12 @@ topology_template: type: tosca.nodes.Container.Application.Docker properties: environment: - IM_DATA_DB: { concat: [ "mysql://root:my-secret@", { get_attribute: [ mysql_container, endpoints, 0 ] } ] } + IM_DATA_DB: + concat: + - "mysql://root:" + - { get_input: mysql_root_password } + - "@" + - { get_attribute: [ mysql_container, endpoints, 0 ] } requirements: - host: im_runtime artifacts: @@ -55,7 +65,7 @@ topology_template: type: tosca.nodes.Container.Application.Docker properties: environment: - MYSQL_ROOT_PASSWORD: my-secret + MYSQL_ROOT_PASSWORD: { get_input: mysql_root_password } requirements: - host: mysql_runtime artifacts: From 1d7b4feb48036a787fd13b903544926684098318 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 02:24:46 +0100 Subject: [PATCH 05/12] Mimor change in test --- test/files/tosca_k8s.yml | 2 +- test/unit/Tosca.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 691b64be..5422e5c2 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -6,6 +6,7 @@ imports: description: TOSCA test for K8s metadata: + # Enable to set the K8s namespace for this deployment namespace: some_namespace infra_name: some_infra @@ -45,7 +46,6 @@ topology_template: my_image: file: grycap/im type: tosca.artifacts.Deployment.Image.Container.Docker - repository: docker_hub # The properties of the runtime to host the container im_runtime: diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 63854b4d..7639b322 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -449,7 +449,7 @@ def test_tosca_k8s(self): self.assertEqual(conf.recipes, None) node = radl.get_system_by_name('im_container') - self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/grycap/im") + self.assertEqual(node.getValue("disk.0.image.url"), "docker://grycap/im") net = radl.get_network_by_id('im_container_pub') self.assertEqual(net.getValue("outports"), '30880/tcp-8800/tcp') self.assertEqual(net.getValue("outbound"), 'yes') From 1567afafb27fdb08ad027a6f9b487228e604e4f7 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 03:49:59 +0100 Subject: [PATCH 06/12] Fix error --- IM/tosca/Tosca.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 8704cd56..5c08729b 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -287,8 +287,10 @@ def to_radl(self, inf_info=None): namespace = self.tosca.tpl.get('metadata', {}).get('namespace') if infra_name or namespace: radl.description = description('desc') - radl.description.setValue('name', infra_name) - radl.description.setValue('namespace', namespace) + if infra_name: + radl.description.setValue('name', infra_name) + if namespace: + radl.description.setValue('namespace', namespace) return all_removal_list, self._complete_radl_networks(radl) From 3e3f1b70059c297f911a29b88d7261624110fe6d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 04:09:17 +0100 Subject: [PATCH 07/12] Mimor change in test --- test/files/tosca_k8s.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 5422e5c2..5b1fbdc4 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -84,7 +84,6 @@ topology_template: mem_size: 2 GB expose_ports: - protocol: tcp - target: 3306 source: 3306 volumes: - "some_vol:/some/path" From 95a7ed0a848cddc65ca4b6b4b282f8fc036d4cea Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 04:18:57 +0100 Subject: [PATCH 08/12] Mimor change in test --- IM/tosca/Tosca.py | 11 ++++++++--- test/files/tosca_k8s.yml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 5c08729b..c7a5b534 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -439,14 +439,19 @@ def _format_outports(ports): if "source_range" in port: source_range = port["source_range"] else: + remote_port = None + local_port = None if "source" in port: remote_port = port["source"] - else: - raise Exception("source port must be specified in PortSpec type.") if "target" in port: local_port = port["target"] - else: + elif remote_port: local_port = remote_port + if not remote_port and local_port: + remote_port = local_port + + if not remote_port or not local_port: + raise Exception("source or target port must be specified in PortSpec type.") # In case of source_range do not use port mapping only direct ports if source_range: diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 5b1fbdc4..17307b34 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -84,7 +84,7 @@ topology_template: mem_size: 2 GB expose_ports: - protocol: tcp - source: 3306 + target: 3306 volumes: - "some_vol:/some/path" From ec907893e904cb4a6c0b59b6565a63fd61952509 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 04:20:11 +0100 Subject: [PATCH 09/12] Mimor change in test --- IM/tosca/Tosca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index c7a5b534..70754669 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -449,7 +449,7 @@ def _format_outports(ports): local_port = remote_port if not remote_port and local_port: remote_port = local_port - + if not remote_port or not local_port: raise Exception("source or target port must be specified in PortSpec type.") From fd757c3ca88197452eb07589384f72dc9381d715 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 06:31:27 +0100 Subject: [PATCH 10/12] Fix --- IM/connectors/Kubernetes.py | 4 +-- test/files/tosca_k8s.yml | 14 ++++----- test/unit/Tosca.py | 10 +++---- test/unit/connectors/Kubernetes.py | 47 +++++++++++++++--------------- 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 229c3299..5869cb32 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -370,8 +370,8 @@ def _generate_pod_data(self, namespace, name, outports, system, volumes, tags): @staticmethod def _get_namespace(inf): namespace = inf.id - if inf.description and inf.description.getValue('namespace'): - namespace = inf.description.getValue('namespace') + if inf.radl.description and inf.radl.description.getValue('namespace'): + namespace = inf.radl.description.getValue('namespace') return namespace def launch(self, inf, radl, requested_radl, num_vm, auth_data): diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 17307b34..5c6ade8c 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -7,7 +7,7 @@ description: TOSCA test for K8s metadata: # Enable to set the K8s namespace for this deployment - namespace: some_namespace + namespace: somenamespace infra_name: some_infra repositories: @@ -21,7 +21,7 @@ topology_template: image: type: string description: The image to be used in the container - default: "mysql:5.7" + default: "mysql:8" mysql_root_password: type: string @@ -53,8 +53,8 @@ topology_template: capabilities: host: properties: - num_cpus: 1 - mem_size: 2 GB + num_cpus: 0.5 + mem_size: 1 GB publish_ports: - protocol: tcp target: 8800 @@ -80,13 +80,13 @@ topology_template: capabilities: host: properties: - num_cpus: 1 - mem_size: 2 GB + num_cpus: 0.5 + mem_size: 1 GB expose_ports: - protocol: tcp target: 3306 volumes: - - "some_vol:/some/path" + - "some_vol:/var/lib/mysql" some_vol: type: tosca.nodes.BlockStorage diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 7639b322..0c5428b8 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -434,13 +434,13 @@ def test_tosca_k8s(self): radl = parse_radl(str(radl)) radl.check() - self.assertEqual(radl.description.getValue("namespace"), "some_namespace") + self.assertEqual(radl.description.getValue("namespace"), "somenamespace") node = radl.get_system_by_name('mysql_container') - self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/mysql:5.7") - self.assertEqual(node.getValue("cpu.count"), 1.0) - self.assertEqual(node.getValue("memory.size"), 2000000000) + self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/mysql:8") + self.assertEqual(node.getValue("cpu.count"), 0.5) + self.assertEqual(node.getValue("memory.size"), 1000000000) self.assertEqual(node.getValue("disk.1.size"), 10000000000) - self.assertEqual(node.getValue("disk.1.mount_path"), '/some/path') + self.assertEqual(node.getValue("disk.1.mount_path"), '/var/lib/mysql') self.assertEqual(node.getValue("environment.variables"), 'MYSQL_ROOT_PASSWORD=my-secret') net = radl.get_network_by_id('mysql_container_priv') self.assertEqual(net.getValue("outports"), '3306/tcp-3306/tcp') diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 9fbe938c..48ac36d7 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -82,30 +82,30 @@ def get_response(self, method, url, verify, headers, data): resp.text = '{"versions": "v1"}' elif url.endswith("/pods/1"): resp.status_code = 200 - resp.text = ('{"metadata": {"namespace":"some_namespace", "name": "name"}, "status": ' + resp.text = ('{"metadata": {"namespace":"somenamespace", "name": "name"}, "status": ' '{"phase":"Running", "hostIP": "158.42.1.1", "podIP": "10.0.0.1"}, ' '"spec": {"containers": [{"image": "image:1.0"}], ' '"volumes": [{"persistentVolumeClaim": {"claimName" : "cname"}}]}}') - if url == "/api/v1/namespaces/some_namespace": + if url == "/api/v1/namespaces/somenamespace": resp.status_code = 200 elif method == "POST": if url.endswith("/pods"): resp.status_code = 201 - resp.text = '{"metadata": {"namespace":"some_namespace", "name": "name"}}' + resp.text = '{"metadata": {"namespace":"somenamespace", "name": "name"}}' elif url.endswith("/services"): resp.status_code = 201 elif url.endswith("/namespaces/"): resp.status_code = 201 elif url.endswith("/persistentvolumeclaims"): resp.status_code = 201 - elif url.endswith("/apis/networking.k8s.io/v1/namespaces/some_namespace/ingresses"): + elif url.endswith("/apis/networking.k8s.io/v1/namespaces/somenamespace/ingresses"): resp.status_code = 201 elif method == "DELETE": if url.endswith("/pods/1"): resp.status_code = 200 elif url.endswith("/services/1"): resp.status_code = 200 - elif url.endswith("/namespaces/some_namespace"): + elif url.endswith("/namespaces/somenamespace"): resp.status_code = 200 elif "persistentvolumeclaims" in url: resp.status_code = 200 @@ -126,7 +126,7 @@ def test_20_launch(self, save_data, requests): radl_data = """ description desc ( name = 'Infrastructure Name' and - namespace = 'some_namespace' + namespace = 'somenamespace' ) network net (outbound = 'yes' and outports = '38080-8080') system test ( @@ -152,8 +152,8 @@ def test_20_launch(self, save_data, requests): inf = MagicMock(["id", "_lock", "add_vm", "description"]) inf.id = "infid" - inf.description = MagicMock(["getValue"]) - inf.description.getValue.return_value = "some_namespace" + inf.radl = radl + inf.description.getValue.return_value = "somenamespace" inf.add_vm.side_effect = self.add_vm res = kube_cloud.launch(inf, radl, radl, 1, auth) success, _ = res[0] @@ -162,14 +162,14 @@ def test_20_launch(self, save_data, requests): exp_pvc = { "apiVersion": "v1", "kind": "PersistentVolumeClaim", - "metadata": {"name": "test-1", "namespace": "some_namespace"}, + "metadata": {"name": "test-1", "namespace": "somenamespace"}, "spec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": 10737418240}}, }, } self.assertEqual(requests.call_args_list[1][0][1], - 'http://server.com:8080/api/v1/namespaces/some_namespace/persistentvolumeclaims') + 'http://server.com:8080/api/v1/namespaces/somenamespace/persistentvolumeclaims') self.assertEqual(json.loads(requests.call_args_list[1][1]['data']), exp_pvc) exp_pod = { @@ -177,7 +177,7 @@ def test_20_launch(self, save_data, requests): "kind": "Pod", "metadata": { "name": "test", - "namespace": "some_namespace", + "namespace": "somenamespace", "labels": {"name": "test", "IM_INFRA_ID": "infid", "key": "invalid_"}, }, "spec": { @@ -202,7 +202,7 @@ def test_20_launch(self, save_data, requests): }, } self.assertEqual(requests.call_args_list[2][0][1], - 'http://server.com:8080/api/v1/namespaces/some_namespace/pods') + 'http://server.com:8080/api/v1/namespaces/somenamespace/pods') self.assertEqual(json.loads(requests.call_args_list[2][1]['data']), exp_pod) exp_svc = { @@ -210,7 +210,7 @@ def test_20_launch(self, save_data, requests): "kind": "Service", "metadata": { "name": "test", - "namespace": "some_namespace", + "namespace": "somenamespace", "labels": {"name": "test"}, }, "spec": { @@ -228,7 +228,7 @@ def test_20_launch(self, save_data, requests): }, } self.assertEqual(requests.call_args_list[3][0][1], - 'http://server.com:8080/api/v1/namespaces/some_namespace/services') + 'http://server.com:8080/api/v1/namespaces/somenamespace/services') self.assertEqual(json.loads(requests.call_args_list[3][1]['data']), exp_svc) exp_ing = { @@ -237,7 +237,7 @@ def test_20_launch(self, save_data, requests): "metadata": { "labels": {"name": "test"}, "name": "test", - "namespace": "some_namespace", + "namespace": "somenamespace", }, "spec": { "rules": [ @@ -263,7 +263,7 @@ def test_20_launch(self, save_data, requests): } self.assertEqual(requests.call_args_list[5][0][1], - 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/some_namespace/ingresses') + 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/somenamespace/ingresses') self.assertEqual(json.loads(requests.call_args_list[5][1]['data']), exp_ing) self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @@ -341,8 +341,9 @@ def test_60_finalize(self, requests): inf = MagicMock() inf.id = "infid" - inf.description = MagicMock(["getValue"]) - inf.description.getValue.return_value = "some_namespace" + inf.radl = MagicMock() + inf.radl.description = MagicMock(["getValue"]) + inf.radl.description.getValue.return_value = "somenamespace" vm = VirtualMachine(inf, "1", kube_cloud.cloud, "", "", kube_cloud, 1) requests.side_effect = self.get_response @@ -351,19 +352,19 @@ def test_60_finalize(self, requests): self.assertEqual(requests.call_args_list[1][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/some_namespace/persistentvolumeclaims/cname')) + 'http://server.com:8080/api/v1/namespaces/somenamespace/persistentvolumeclaims/cname')) self.assertEqual(requests.call_args_list[2][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/some_namespace/pods/1')) + 'http://server.com:8080/api/v1/namespaces/somenamespace/pods/1')) self.assertEqual(requests.call_args_list[3][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/some_namespace/services/1')) + 'http://server.com:8080/api/v1/namespaces/somenamespace/services/1')) self.assertEqual(requests.call_args_list[4][0], ('DELETE', - 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/some_namespace/ingresses/1')) + 'http://server.com:8080/apis/networking.k8s.io/v1/namespaces/somenamespace/ingresses/1')) self.assertEqual(requests.call_args_list[5][0], ('DELETE', - 'http://server.com:8080/api/v1/namespaces/some_namespace')) + 'http://server.com:8080/api/v1/namespaces/somenamespace')) self.assertTrue(success, msg="ERROR: finalizing VM info.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) From b2a19698a4fc8f050f422f051e36bc159004db08 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 08:03:48 +0100 Subject: [PATCH 11/12] Fix --- IM/VirtualMachine.py | 7 +++-- IM/connectors/Kubernetes.py | 48 +++++++++++++++++++----------- IM/tosca/Tosca.py | 4 +-- test/files/tosca_k8s.yml | 2 ++ test/unit/Tosca.py | 4 +-- test/unit/connectors/Kubernetes.py | 1 - 6 files changed, 41 insertions(+), 25 deletions(-) diff --git a/IM/VirtualMachine.py b/IM/VirtualMachine.py index 1a7883b1..3210365f 100644 --- a/IM/VirtualMachine.py +++ b/IM/VirtualMachine.py @@ -54,6 +54,8 @@ class VirtualMachine(LoggerMixin): SSH_REVERSE_BASE_PORT = 20000 + NO_DNS_NAME_SET = ["Kubernetes", "OSCAR", "Lambda"] + logger = logging.getLogger('InfrastructureManager') def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector=None, im_id=None): @@ -586,8 +588,9 @@ def update_status(self, auth, force=False): self.state = new_state self.info.systems[0].setValue("state", new_state) - # Replace the #N# in dns_names - self.replace_dns_name(self.info.systems[0]) + if self.getCloudConnector().type not in self.NO_DNS_NAME_SET: + # Replace the #N# in dns_names + self.replace_dns_name(self.info.systems[0]) return updated diff --git a/IM/connectors/Kubernetes.py b/IM/connectors/Kubernetes.py index 5869cb32..aa37ca23 100644 --- a/IM/connectors/Kubernetes.py +++ b/IM/connectors/Kubernetes.py @@ -191,9 +191,9 @@ def _create_volumes(self, namespace, system, pod_name, auth_data, persistent=Fal return res - def create_service_data(self, namespace, name, outports, auth_data, vm): + def create_service_data(self, namespace, name, outports, public, auth_data, vm): try: - service_data = self._generate_service_data(namespace, name, outports) + service_data = self._generate_service_data(namespace, name, outports, public) self.log_debug("Creating Service: %s/%s" % (namespace, name)) headers = {'Content-Type': 'application/json'} uri = "/api/v1/namespaces/%s/%s" % (namespace, "services") @@ -217,7 +217,7 @@ def create_service_data(self, namespace, name, outports, auth_data, vm): self.error_messages += "Error creating service to access pod %s" % name self.log_exception("Error creating service.") - def _generate_service_data(self, namespace, name, outports): + def _generate_service_data(self, namespace, name, outports, public): service_data = {'apiVersion': 'v1', 'kind': 'Service'} service_data['metadata'] = { 'name': name, @@ -235,16 +235,19 @@ def _generate_service_data(self, namespace, name, outports): 'protocol': outport.get_protocol().upper(), 'targetPort': outport.get_local_port(), 'name': 'port%s' % outport.get_local_port()} - if outport.get_remote_port(): + if public and outport.get_remote_port(): port['nodePort'] = outport.get_remote_port() ports.append(port) service_data['spec'] = { - 'type': 'NodePort', + 'type': 'ClusterIP', 'ports': ports, 'selector': {'name': name} } + if public: + service_data['spec']['type'] = 'NodePort' + return service_data def create_ingress(self, namespace, name, dns, port, auth_data): @@ -377,15 +380,6 @@ def _get_namespace(inf): def launch(self, inf, radl, requested_radl, num_vm, auth_data): system = radl.systems[0] - public_net = None - for net in radl.networks: - if net.isPublic(): - public_net = net - - outports = None - if public_net: - outports = public_net.getOutPorts() - res = [] # First create the namespace for the infrastructure namespace = self._get_namespace(inf) @@ -419,12 +413,20 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): vm = VirtualMachine(inf, None, self.cloud, radl, requested_radl, self) vm.destroy = True inf.add_vm(vm) - pod_name = system.name + pod_name = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_]', '-', system.name) volumes = self._create_volumes(namespace, system, pod_name, auth_data, True) tags = self.get_instance_tags(system, auth_data, inf) + outports = [] + pub_net = vm.getConnectedNet(public=True) + priv_net = vm.getConnectedNet(public=False) + if pub_net: + outports = pub_net.getOutPorts() + elif priv_net: + outports = priv_net.getOutPorts() + pod_data = self._generate_pod_data(namespace, pod_name, outports, system, volumes, tags) self.log_debug("Creating POD: %s/%s" % (namespace, pod_name)) @@ -441,7 +443,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): else: dns_name = system.getValue("net_interface.0.dns_name") - self.create_service_data(namespace, pod_name, outports, auth_data, vm) + self.create_service_data(namespace, pod_name, outports, pub_net, auth_data, vm) output = json.loads(resp.text) vm.id = output["metadata"]["name"] @@ -480,6 +482,12 @@ def _get_pod(self, vm, auth_data): self.log_exception("Error connecting with Kubernetes API server") return (False, None, "Error connecting with Kubernetes API server: " + str(ex)) + def _get_float_cpu(self, cpu): + if cpu.endswith("m"): + return float(cpu[:-1]) / 1000 + else: + return float(cpu) + def updateVMInfo(self, vm, auth_data): success, status, output = self._get_pod(vm, auth_data) if success: @@ -488,7 +496,7 @@ def updateVMInfo(self, vm, auth_data): pod_limits = output['spec']['containers'][0].get('resources', {}).get('limits') if pod_limits: - vm.info.systems[0].setValue('cpu.count', float(pod_limits['cpu'])) + vm.info.systems[0].setValue('cpu.count', self._get_float_cpu(pod_limits['cpu'])) memory = self.convert_memory_unit(pod_limits['memory'], "B") vm.info.systems[0].setValue('memory.size', memory) @@ -509,7 +517,6 @@ def setIPs(self, vm, pod_info): - vm(:py:class:`IM.VirtualMachine`): VM information. - pod_info(dict): JSON information about the POD """ - public_ips = [] private_ips = [] if 'hostIP' in pod_info["status"]: @@ -522,6 +529,11 @@ def setIPs(self, vm, pod_info): if 'podIP' in pod_info["status"]: private_ips = [str(pod_info["status"]["podIP"])] + if not vm.getConnectedNet(public=True): + public_ips = [] + if not vm.getConnectedNet(public=False): + private_ips = [] + vm.setIps(public_ips, private_ips) def finalize(self, vm, last, auth_data): diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 70754669..38cf7d9c 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -1379,7 +1379,7 @@ def _get_attribute_result(self, func, node, inf_info): res.append(url + ":%s" % outport.get_remote_port()) if priv_net: # set the internal DNS name - url = "%s.%s" % (vm.info.systems[0].name, inf_info.id) + url = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_]', '-', node.name) if priv_net.getOutPorts(): for outport in priv_net.getOutPorts(): res.append(url + ":%s" % outport.get_local_port()) @@ -1474,7 +1474,7 @@ def _get_attribute_result(self, func, node, inf_info): outport.get_remote_port()) else: # set the internal DNS name - url = node.name + url = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_]', '-', node.name) if net.getOutPorts(): for outport in net.getOutPorts(): res.append(url + ":%s" % outport.get_local_port()) diff --git a/test/files/tosca_k8s.yml b/test/files/tosca_k8s.yml index 5c6ade8c..38efc17e 100644 --- a/test/files/tosca_k8s.yml +++ b/test/files/tosca_k8s.yml @@ -40,6 +40,7 @@ topology_template: - { get_input: mysql_root_password } - "@" - { get_attribute: [ mysql_container, endpoints, 0 ] } + - "/im-db" requirements: - host: im_runtime artifacts: @@ -66,6 +67,7 @@ topology_template: properties: environment: MYSQL_ROOT_PASSWORD: { get_input: mysql_root_password } + MYSQL_DATABASE: "im-db" requirements: - host: mysql_runtime artifacts: diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 0c5428b8..1c3fd9e2 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -441,7 +441,7 @@ def test_tosca_k8s(self): self.assertEqual(node.getValue("memory.size"), 1000000000) self.assertEqual(node.getValue("disk.1.size"), 10000000000) self.assertEqual(node.getValue("disk.1.mount_path"), '/var/lib/mysql') - self.assertEqual(node.getValue("environment.variables"), 'MYSQL_ROOT_PASSWORD=my-secret') + self.assertEqual(node.getValue("environment.variables"), 'MYSQL_ROOT_PASSWORD=my-secret,MYSQL_DATABASE=im-db') net = radl.get_network_by_id('mysql_container_priv') self.assertEqual(net.getValue("outports"), '3306/tcp-3306/tcp') self.assertEqual(net.getValue("outbound"), 'no') @@ -454,7 +454,7 @@ def test_tosca_k8s(self): self.assertEqual(net.getValue("outports"), '30880/tcp-8800/tcp') self.assertEqual(net.getValue("outbound"), 'yes') self.assertEqual(node.getValue("environment.variables"), - 'IM_DATA_DB=mysql://root:my-secret@mysql_container:3306') + 'IM_DATA_DB=mysql://root:my-secret@mysql-container:3306/im-db') conf = radl.get_configure_by_name('im_container') self.assertEqual(conf.recipes, None) diff --git a/test/unit/connectors/Kubernetes.py b/test/unit/connectors/Kubernetes.py index 48ac36d7..c91541a8 100755 --- a/test/unit/connectors/Kubernetes.py +++ b/test/unit/connectors/Kubernetes.py @@ -296,7 +296,6 @@ def test_30_updateVMInfo(self, requests): self.assertTrue(success, msg="ERROR: updating VM info.") self.assertEqual(vm.info.systems[0].getValue("net_interface.0.ip"), "158.42.1.1") - self.assertEqual(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.1") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) @patch('requests.request') From 74e8de3a8da3fbb873521e0fc841ea4092f39ca9 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 20 Feb 2024 08:13:04 +0100 Subject: [PATCH 12/12] Fix --- test/unit/Tosca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 1c3fd9e2..fb151c85 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -479,7 +479,7 @@ def test_tosca_k8s_get_attribute(self): inf.vm_list = [vm, vm2] outputs = tosca.get_outputs(inf) self.assertEqual(outputs, {'im_service_endpoint': '8.8.8.8:30880', - 'mysql_service_endpoint': 'mysql_container.%s:3306' % inf.id}) + 'mysql_service_endpoint': 'mysql-container:3306'}) if __name__ == "__main__":