Merge pull request #1529 from grycap/new_k8s

New k8s

IM/VirtualMachine.py

@@ -54,6 +54,8 @@ class VirtualMachine(LoggerMixin):
 
     SSH_REVERSE_BASE_PORT = 20000
 
+    NO_DNS_NAME_SET = ["Kubernetes", "OSCAR", "Lambda"]
+
     logger = logging.getLogger('InfrastructureManager')
 
     def __init__(self, inf, cloud_id, cloud, info, requested_radl, cloud_connector=None, im_id=None):
@@ -267,6 +269,16 @@ def getPrivateIP(self):
         """
         return self.info.getPrivateIP()
 
+    def getConnectedNet(self, public=False):
+        """
+        Return the first public/private net connected to this VM if available
+        """
+        for netid in self.info.systems[0].getNetworkIDs():
+            net = self.info.get_network_by_id(netid)
+            if net.isPublic() == public:
+                return net
+        return None
+
     def getNumNetworkIfaces(self):
         """
         Get the number of net interfaces of this VM
@@ -576,8 +588,9 @@ def update_status(self, auth, force=False):
             self.state = new_state
             self.info.systems[0].setValue("state", new_state)
 
-            # Replace the #N# in dns_names
-            self.replace_dns_name(self.info.systems[0])
+            if self.getCloudConnector().type not in self.NO_DNS_NAME_SET:
+                # Replace the #N# in dns_names
+                self.replace_dns_name(self.info.systems[0])
 
         return updated
 

IM/tosca/Tosca.py

@@ -284,9 +284,13 @@ def to_radl(self, inf_info=None):
         self._check_private_networks(radl, inf_info)
 
         infra_name = self.tosca.tpl.get('metadata', {}).get('infra_name')
-        if infra_name:
+        namespace = self.tosca.tpl.get('metadata', {}).get('namespace')
+        if infra_name or namespace:
             radl.description = description('desc')
-            radl.description.setValue('name', infra_name)
+            if infra_name:
+                radl.description.setValue('name', infra_name)
+            if namespace:
+                radl.description.setValue('namespace', namespace)
 
         return all_removal_list, self._complete_radl_networks(radl)
 
@@ -435,14 +439,19 @@ def _format_outports(ports):
             if "source_range" in port:
                 source_range = port["source_range"]
             else:
+                remote_port = None
+                local_port = None
                 if "source" in port:
                     remote_port = port["source"]
-                else:
-                    raise Exception("source port must be specified in PortSpec type.")
                 if "target" in port:
                     local_port = port["target"]
-                else:
+                elif remote_port:
                     local_port = remote_port
+                if not remote_port and local_port:
+                    remote_port = local_port
+
+                if not remote_port or not local_port:
+                    raise Exception("source or target port must be specified in PortSpec type.")
 
             # In case of source_range do not use port mapping only direct ports
             if source_range:
@@ -1206,7 +1215,8 @@ def _get_attribute_result(self, func, node, inf_info):
         root_type = Tosca._get_root_parent_type(node).type
 
         host_node = self._find_host_node(node, self.tosca.nodetemplates)
-        if root_type == "tosca.nodes.aisprint.FaaS.Function" and host_node is None:
+        if (host_node is None and (root_type == "tosca.nodes.aisprint.FaaS.Function" or
+                                   node.type == "tosca.nodes.Container.Application.Docker")):
             # in case of FaaS functions without host, the node is the host
             host_node = node
 
@@ -1352,6 +1362,33 @@ def _get_attribute_result(self, func, node, inf_info):
 
                 Tosca.logger.warning("Attribute credential only supported in tosca.nodes.aisprint.FaaS.Function")
                 return None
+            elif attribute_name == "endpoints":
+                if node.type == "tosca.nodes.Container.Application.Docker":
+                    res = []
+                    dmsname = vm.info.systems[0].getValue("net_interface.0.dns_name")
+                    pub_net = vm.getConnectedNet(public=True)
+                    priv_net = vm.getConnectedNet(public=False)
+                    if pub_net:
+                        url = "%s" % vm.getPublicIP()
+                        if pub_net.getOutPorts():
+                            for outport in pub_net.getOutPorts():
+                                # if DNS name is set, the endpoint of the first public port is the DNS name
+                                if dmsname and not res:
+                                    res.append("%s" % dmsname)
+                                else:
+                                    res.append(url + ":%s" % outport.get_remote_port())
+                    if priv_net:
+                        # set the internal DNS name
+                        url = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_]', '-', node.name)
+                        if priv_net.getOutPorts():
+                            for outport in priv_net.getOutPorts():
+                                res.append(url + ":%s" % outport.get_local_port())
+
+                    if index is not None:
+                        res = res[index]
+                    return res
+                Tosca.logger.warning("Attribute credential only supported in tosca.nodes.Container.Application.Docker")
+                return None
             else:
                 Tosca.logger.warning("Attribute %s not supported." % attribute_name)
                 return None
@@ -1413,6 +1450,40 @@ def _get_attribute_result(self, func, node, inf_info):
 
                 Tosca.logger.warning("Attribute endpoint only supported in tosca.nodes.aisprint.FaaS.Function")
                 return None
+            elif attribute_name == "endpoints":
+                if node.type == "tosca.nodes.Container.Application.Docker":
+                    k8s_sys, nets = self._gen_k8s_system(node, self.tosca.nodetemplates)
+                    if nets:
+                        net = nets[0]
+                    else:
+                        if index is not None:
+                            return None
+                        else:
+                            return []
+
+                    res = []
+                    if net.isPublic():
+                        dmsname = k8s_sys.getValue("net_interface.0.dns_name")
+                        if net.getOutPorts():
+                            for outport in net.getOutPorts():
+                                # if DNS name is set, the endpoint of the first public port is the DNS name
+                                if dmsname and not res:
+                                    res.append("%s" % dmsname)
+                                else:
+                                    res.append("{{ hostvars[groups['%s'][0]]['IM_NODE_PUBLIC_IP'] }}:%s" %
+                                               outport.get_remote_port())
+                    else:
+                        # set the internal DNS name
+                        url = re.sub('[!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~_]', '-', node.name)
+                        if net.getOutPorts():
+                            for outport in net.getOutPorts():
+                                res.append(url + ":%s" % outport.get_local_port())
+
+                    if index is not None:
+                        res = res[index]
+                    return res
+                Tosca.logger.warning("Attribute credential only supported in tosca.nodes.Container.Application.Docker")
+                return None
             else:
                 Tosca.logger.warning("Attribute %s not supported." % attribute_name)
                 return None
@@ -2210,10 +2281,13 @@ def _gen_k8s_system(self, node, nodetemplates):
                         pub.setValue("outbound", "yes")
                         pub.setValue("outports", self._format_outports(value))
                         nets.append(pub)
+                        res.setValue("net_interface.0.connection", "%s_pub" % node.name)
                     elif prop.name == 'expose_ports':
                         # Asume that publish_ports must be published as ClusterIP
                         priv = network("%s_priv" % node.name)
+                        priv.setValue("outbound", "no")
                         priv.setValue("outports", self._format_outports(value))
                         nets.append(priv)
+                        res.setValue("net_interface.0.connection", "%s_priv" % node.name)
 
         return res, nets

requirements-tests.txt

@@ -35,3 +35,4 @@ scar
 mock
 coverage
 requests-cache
+packaging

setup.py

@@ -70,5 +70,5 @@
       install_requires=["ansible >=2.4", "paramiko >= 1.14", "PyYAML", suds_pkg, sqlite_pkg, "cheroot",
                         "boto >= 2.29", "apache-libcloud >= 3.2.0", "RADL >= 1.1.5", "bottle", "netaddr",
                         "requests >= 2.19", "scp", "tosca-parser", 'defusedxml', 'urllib3>=1.23', 'hvac',
-                        'psutil', 'scar', 'requests-cache >= 1.0.0']
+                        'psutil', 'scar', 'requests-cache >= 1.0.0', 'packaging']
       )

test/files/tosca_k8s.yml

@@ -5,6 +5,11 @@ imports:
 
 description: TOSCA test for K8s
 
+metadata:
+  # Enable to set the K8s namespace for this deployment
+  namespace: somenamespace
+  infra_name: some_infra
+
 repositories:
 
   docker_hub: docker.io
@@ -16,16 +21,53 @@ topology_template:
     image:
       type: string
       description: The image to be used in the container
-      default: "mysql:5.7"
+      default: "mysql:8"
+
+    mysql_root_password:
+      type: string
+      description: The root password for the MySQL container
+      default: "my-secret"
 
   node_templates:
 
+    im_container:
+      type: tosca.nodes.Container.Application.Docker
+      properties:
+        environment:
+          IM_DATA_DB:
+            concat:
+              - "mysql://root:"
+              - { get_input: mysql_root_password }
+              - "@"
+              - { get_attribute: [ mysql_container, endpoints, 0 ] }
+              - "/im-db"
+      requirements:
+        - host: im_runtime
+      artifacts:
+        my_image:
+          file: grycap/im
+          type: tosca.artifacts.Deployment.Image.Container.Docker
+
+    # The properties of the runtime to host the container
+    im_runtime:
+      type: tosca.nodes.Container.Runtime.Docker
+      capabilities:
+        host:
+          properties:
+            num_cpus: 0.5
+            mem_size: 1 GB
+            publish_ports:
+              - protocol: tcp
+                target: 8800
+                source: 30880
+
     # The MYSQL container based on official MySQL image in Docker hub
     mysql_container:
       type: tosca.nodes.Container.Application.Docker
       properties:
         environment:
-          MYSQL_ROOT_PASSWORD: my-secret
+          MYSQL_ROOT_PASSWORD: { get_input: mysql_root_password }
+          MYSQL_DATABASE: "im-db"
       requirements:
         - host: mysql_runtime
       artifacts:
@@ -40,14 +82,13 @@ topology_template:
       capabilities:
         host:
           properties:
-            num_cpus: 1
-            mem_size: 2 GB
-            publish_ports:
+            num_cpus: 0.5
+            mem_size: 1 GB
+            expose_ports:
               - protocol: tcp
                 target: 3306
-                source: 33306
             volumes:
-              - "some_vol:/some/path"
+              - "some_vol:/var/lib/mysql"
 
     some_vol:
       type: tosca.nodes.BlockStorage
@@ -56,3 +97,8 @@ topology_template:
         # Set the PV name in this field
         # volume_id: "PV name"
 
+  outputs:
+    im_service_endpoint:
+      value: { get_attribute: [ im_container, endpoints, 0 ] }
+    mysql_service_endpoint:
+      value: { get_attribute: [ mysql_container, endpoints, 0 ] }

test/unit/Tosca.py

@@ -434,19 +434,53 @@ def test_tosca_k8s(self):
         radl = parse_radl(str(radl))
         radl.check()
 
+        self.assertEqual(radl.description.getValue("namespace"), "somenamespace")
         node = radl.get_system_by_name('mysql_container')
-        self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/mysql:5.7")
-        self.assertEqual(node.getValue("cpu.count"), 1.0)
-        self.assertEqual(node.getValue("memory.size"), 2000000000)
+        self.assertEqual(node.getValue("disk.0.image.url"), "docker://docker.io/mysql:8")
+        self.assertEqual(node.getValue("cpu.count"), 0.5)
+        self.assertEqual(node.getValue("memory.size"), 1000000000)
         self.assertEqual(node.getValue("disk.1.size"), 10000000000)
-        self.assertEqual(node.getValue("disk.1.mount_path"), '/some/path')
-        self.assertEqual(node.getValue("environment.variables"), 'MYSQL_ROOT_PASSWORD=my-secret')
-        net = radl.get_network_by_id('mysql_container_pub')
-        self.assertEqual(net.getValue("outports"), '33306/tcp-3306/tcp')
-        self.assertEqual(net.getValue("outbound"), 'yes')
+        self.assertEqual(node.getValue("disk.1.mount_path"), '/var/lib/mysql')
+        self.assertEqual(node.getValue("environment.variables"), 'MYSQL_ROOT_PASSWORD=my-secret,MYSQL_DATABASE=im-db')
+        net = radl.get_network_by_id('mysql_container_priv')
+        self.assertEqual(net.getValue("outports"), '3306/tcp-3306/tcp')
+        self.assertEqual(net.getValue("outbound"), 'no')
         conf = radl.get_configure_by_name('mysql_container')
         self.assertEqual(conf.recipes, None)
 
+        node = radl.get_system_by_name('im_container')
+        self.assertEqual(node.getValue("disk.0.image.url"), "docker://grycap/im")
+        net = radl.get_network_by_id('im_container_pub')
+        self.assertEqual(net.getValue("outports"), '30880/tcp-8800/tcp')
+        self.assertEqual(net.getValue("outbound"), 'yes')
+        self.assertEqual(node.getValue("environment.variables"),
+                         'IM_DATA_DB=mysql://root:my-secret@mysql-container:3306/im-db')
+        conf = radl.get_configure_by_name('im_container')
+        self.assertEqual(conf.recipes, None)
+
+    def test_tosca_k8s_get_attribute(self):
+        """Test TOSCA K8s get_attributes function"""
+        tosca_data = read_file_as_string('../files/tosca_k8s.yml')
+        tosca = Tosca(tosca_data)
+        _, radl = tosca.to_radl()
+        radl1 = radl.clone()
+        radl1.systems = [radl.get_system_by_name('im_container')]
+        inf = InfrastructureInfo()
+        radl1.systems[0].setValue("net_interface.0.ip", "8.8.8.8")
+
+        radl2 = radl.clone()
+        radl2.systems = [radl.get_system_by_name('mysql_container')]
+
+        cloud_info = MagicMock()
+        vm = VirtualMachine(inf, "1", cloud_info, radl1, radl1, None)
+        vm2 = VirtualMachine(inf, "2", cloud_info, radl2, radl2, None)
+        vm.requested_radl = radl1
+        vm2.requested_radl = radl2
+        inf.vm_list = [vm, vm2]
+        outputs = tosca.get_outputs(inf)
+        self.assertEqual(outputs, {'im_service_endpoint': '8.8.8.8:30880',
+                                   'mysql_service_endpoint': 'mysql-container:3306'})
+
 
 if __name__ == "__main__":
     unittest.main()