Merge branch 'egi_conn' into devel

IM/AppDB.py

@@ -102,7 +102,7 @@ def get_image_id(site_id, image_name, vo_name):
         return None
 
     @staticmethod
-    def get_image_data(str_url, stype="occi"):
+    def get_image_data(str_url, stype="occi", vo=None):
         """
         The url has this format: appdb://UPV-GRyCAP/egi.docker.ubuntu.16.04?fedcloud.egi.eu
         or this one appdb://UPV-GRyCAP/83d5e854-a128-5b1f-9457-d32e10a720a6:8135
@@ -130,6 +130,8 @@ def get_image_data(str_url, stype="occi"):
                     return None, None, "No image ID returned from EGI AppDB for image: %s/%s." % (site_id,
                                                                                                   image_name)
             else:
+                if not vo_name:
+                    vo_name = vo
                 image_id = AppDB.get_image_id(site_id, image_name, vo_name)
                 if not image_id:
                     return None, None, "No image ID returned from EGI AppDB for image: %s/%s/%s." % (site_id,
@@ -161,3 +163,24 @@ def get_image_id_from_uri(site_id, image_mp_uri):
                             return image_basename
 
         return None
+
+    @staticmethod
+    def get_project_ids(site_id):
+        projects = {}
+        # Until it is on the prod instance use the Devel one
+
+        data = AppDB.appdb_call('/rest/1.0/va_providers/%s' % site_id)
+        if (data and 'virtualization:provider' in data and data['virtualization:provider'] and
+                'provider:shares' in data['virtualization:provider'] and
+                data['virtualization:provider']['provider:shares'] and
+                'vo:vo' in data['virtualization:provider']['provider:shares'] and
+                data['virtualization:provider']['provider:shares']['vo:vo']):
+            if isinstance(data['virtualization:provider']['provider:shares']['vo:vo'], list):
+                shares = data['virtualization:provider']['provider:shares']['vo:vo']
+            else:
+                shares = [data['virtualization:provider']['provider:shares']['vo:vo']]
+
+            for vo in shares:
+                projects[vo["#text"]] = vo['@projectid']
+
+        return projects

IM/connectors/EGI.py

@@ -0,0 +1,136 @@
+# IM - Infrastructure Manager
+# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from IM.connectors.OpenStack import OpenStackCloudConnector
+from IM.AppDB import AppDB
+from IM.auth import Authentication
+from IM.CloudInfo import CloudInfo
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+
+class EGICloudConnector(OpenStackCloudConnector):
+    """
+    Cloud Launcher to EGI using LibCloud
+    """
+
+    type = "EGI"
+    """str with the name of the provider."""
+
+    def __init__(self, cloud_info, inf):
+        self.egi_auth = None
+        OpenStackCloudConnector.__init__(self, cloud_info, inf)
+
+    def get_driver(self, auth_data):
+        """
+        Get the driver from the auth data
+
+        Arguments:
+                - auth(Authentication): parsed authentication tokens.
+
+        Returns: a :py:class:`libcloud.compute.base.NodeDriver` or None in case of error
+        """
+        auths = auth_data.getAuthInfo(self.type, self.cloud.server)
+        if not auths:
+            raise Exception("No auth data has been specified to EGI.")
+        else:
+            auth = auths[0]
+
+        if self.driver and self.egi_auth.compare(auth_data, self.type, self.cloud.server):
+            return self.driver
+        else:
+            self.egi_auth = auth_data
+
+            if 'host' in auth and 'vo' in auth and 'token' in auth:
+                ost_auth = {'id': auth['id'], 'type': 'OpenStack', 'username': 'egi.eu', 'tenant': 'openid',
+                            'password': auth['token'], 'auth_version': '3.x_oidc_access_token', 'vo': auth['vo']}
+                site_id = AppDB.get_site_id(auth["host"], stype="openstack")
+                site_url = AppDB.get_site_url(site_id)
+                if not site_url:
+                    raise Exception("Invalid site name '%s'. Not found at AppDB." % auth['host'])
+                ost_auth['host'] = site_url
+                projects = AppDB.get_project_ids(site_id)
+                # If the VO does not appear in the project IDs
+                if auth['vo'] in projects:
+                    ost_auth['domain'] = projects[auth['vo']]
+                else:
+                    # let's use the VO name directly
+                    ost_auth['domain'] = auth['vo']
+
+                if 'api_version' in auth:
+                    ost_auth['api_version'] = auth['api_version']
+
+                new_auth = Authentication([ost_auth])
+
+                orig_cloud = self.cloud
+                self.cloud = CloudInfo.get_cloud_list(new_auth)[0]
+                self.type = OpenStackCloudConnector.type
+                driver = OpenStackCloudConnector.get_driver(self, new_auth)
+                self.type = EGICloudConnector.type
+                self.cloud = orig_cloud
+
+                self.driver = driver
+                return driver
+            else:
+                self.log_error("No correct auth data has been specified to EGI: host, vo, and token")
+                raise Exception("No correct auth data has been specified to EG: host, vo, and token")
+
+    def concrete_system(self, radl_system, str_url, auth_data):
+        url = urlparse(str_url)
+        protocol = url[0]
+        src_host = url[1].split(':')[0]
+
+        if protocol in ["ost", "appdb"] and self.cloud.server and not self.cloud.protocol:
+            site_host = ""
+            if protocol == "ost":
+                site_url = AppDB.get_site_url(self.cloud.server)
+                site_host = urlparse(site_url)[1].split(':')[0]
+
+            if ((protocol == "ost" and site_host == src_host) or
+                    (protocol == "appdb" and src_host == self.cloud.server)):
+                driver = self.get_driver(auth_data)
+
+                res_system = radl_system.clone()
+                instance_type = self.get_instance_type(driver, res_system)
+                self.update_system_info_from_instance(res_system, instance_type)
+
+                username = res_system.getValue('disk.0.os.credentials.username')
+                if not username:
+                    res_system.setValue('disk.0.os.credentials.username', self.DEFAULT_USER)
+
+                return res_system
+            else:
+                return None
+        else:
+            return None
+
+    @staticmethod
+    def _get_tenant_id(driver, auth):
+        """
+        Workaround function to get tenant id from tenant name
+        """
+        if 'auth_version' in auth and auth['auth_version'] == '3.x_oidc_access_token':
+            return auth['domain']
+        else:
+            if 'tenant_id' in auth:
+                return auth['tenant_id']
+            else:
+                return auth['tenant']
+
+        return None

IM/connectors/OpenStack.py

@@ -1052,14 +1052,24 @@ def get_volumes(driver, image, radl):
 
         return res
 
+    def get_vo_name(self, auth_data):
+        """Get vo field from auth data if present"""
+        auths = auth_data.getAuthInfo(self.type, self.cloud.server)
+        if auths:
+            auth = auths[0]
+            if 'vo' in auth and auth['vo']:
+                return auth['vo']
+        return None
+
     def launch(self, inf, radl, requested_radl, num_vm, auth_data):
         driver = self.get_driver(auth_data)
 
         system = radl.systems[0]
 
         image_url = system.getValue("disk.0.image.url")
         if urlparse(image_url)[0] == "appdb":
-            _, image_id, msg = AppDB.get_image_data(image_url, "openstack")
+            vo = self.get_vo_name(auth_data)
+            _, image_id, msg = AppDB.get_image_data(image_url, "openstack", vo)
             if not image_id:
                 self.log_error(msg)
                 raise Exception("Error in appdb image: %s" % msg)

IM/connectors/__init__.py

@@ -15,5 +15,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-__all__ = ['CloudConnector', 'EC2', 'OCCI', 'OpenNebula', 'OpenStack', 'AzureClassic', 'Docker', 'GCE',
-           'FogBow', 'Azure', 'DeployedNode', 'Kubernetes', 'Dummy', 'vSphere', 'CloudStack', 'Linode', 'Orange']
+__all__ = ['CloudConnector', 'EC2', 'OCCI', 'OpenNebula', 'OpenStack', 'AzureClassic', 'Docker', 'GCE', 'FogBow',
+           'Azure', 'DeployedNode', 'Kubernetes', 'Dummy', 'vSphere', 'CloudStack', 'Linode', 'Orange', 'EGI']

doc/source/client.rst

@@ -204,7 +204,7 @@ The available keys are:
 * ``type`` indicates the service that refers the credential. The services
   supported are ``InfrastructureManager``, ``VMRC``, ``OpenNebula``, ``EC2``,, ``FogBow``, 
   ``OpenStack``, ``OCCI``, ``LibCloud``, ``Docker``, ``GCE``, ``Azure``, ``AzureClassic``,
-  ``Kubernetes``, ``vSphere``, ``Linode`` and ``Orange``.
+  ``Kubernetes``, ``vSphere``, ``Linode``, ``Orange`` and ``EGI``.
 
 * ``username`` indicates the user name associated to the credential. In EC2
   it refers to the *Access Key ID*. In GCE it refers to *Service Account’s Email Address*. 
@@ -220,6 +220,7 @@ The available keys are:
   This field is only used in the OpenStack and Orange plugins.
 
 * ``host`` indicates the address of the access point to the cloud provider.
+  In case of EGI connector it indicates the site name.
   This field is not used in IM, GCE, Azure, and EC2 credentials.
 
 * ``proxy`` indicates the content of the proxy file associated to the credential.
@@ -247,7 +248,10 @@ The available keys are:
   plugin check the documentation of the Azure python SDK:
   `here <https://docs.microsoft.com/en-us/python/azure/python-sdk-azure-authenticate?view=azure-python>`_
 
-* ``token`` indicates the OpenID token associated to the credential. This field is used in the OCCI plugin (from version 1.6.2). 
+* ``token`` indicates the OpenID token associated to the credential. This field is used in the EGI and OCCI plugins. 
+
+* ``vo`` indicates the VO name associated to the credential. This field is used in the EGI plugin. 
+
 
 OpenStack additional fields
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -341,6 +345,9 @@ So the auth line will be like that::
 
    id = ost; type = OpenStack; host = https://ostserver:5000; username = egi.eu; tenant = openid; password = egi_aai_token_value; auth_version = 3.x_oidc_access_token; domain = project_name
 
+From IM version 1.10.2 the EGI connector is available and you can use this kind of auth line::
+
+   id = egi; type = EGI; host = CESGA; vo = vo.access.egi.eu; token = egi_aai_token_value
 
 Open Telekom Cloud
 ++++++++++++++++++
@@ -397,6 +404,8 @@ An example of the auth file::
    id = linode; type = Linode; username = apikey
    # Orange Flexible Cloud auth data
    id = orange; type = Orange; username = usern; password = pass; domain = DOMAIN; region = region; tenant = tenant
+   #  EGI auth data
+   id = egi; type = EGI; host = SITE_NAME; vo = vo_name; token = egi_aai_token_value
 
 IM Server does not store the credentials used in the creation of
 infrastructures. Then the user has to provide them in every call of

doc/source/manual.rst

@@ -77,7 +77,7 @@ Optional Packages
    In newer versions (9.0 and later) the functionality has been moved `the cheroot
    library<https://github.com/cherrypy/cheroot>`_ it can be installed using pip.
 * `apache-libcloud <http://libcloud.apache.org/>`_ 3.0 or later is used in the
-  LibCloud, OpenStack and GCE connectors.
+  LibCloud, OpenStack, EGI and GCE connectors.
 * `boto <http://boto.readthedocs.org>`_ 2.29.0 or later is used as interface to
   Amazon EC2. It is available as package named ``python-boto`` in Debian based
   distributions. It can also be downloaded from `boto GitHub repository <https://github.com/boto/boto>`_.

doc/source/radl.rst

@@ -302,7 +302,7 @@ machine.  The supported features are:
 
    * ``one://<server>:<port>/<image-id>``, for OpenNebula;
    * ``one://<server>:<port>/<image-name>``, for OpenNebula;
-   * ``ost://<server>:<port>/<image-id>``, for OpenStack;
+   * ``ost://<server>:<port>/<image-id>``, for OpenStack or EGI;
    * ``aws://<region>/<ami-id>``, for Amazon Web Service;
    * ``aws://<region>/<snapshot-id>``, for Amazon Web Service;
    * ``aws://<region>/<snapshot-name>``, for Amazon Web Service;
@@ -311,7 +311,8 @@ machine.  The supported features are:
    * ``azr://<publisher>/<offer>/<sku>/<version>``, for Microsoft Azure;
    * ``azr://[snapshots|disk]/<rgname>/<diskname>``, for Microsoft Azure;
    * ``<fedcloud_endpoint_url>/<image_id>``, for FedCloud OCCI connector.
-   * ``appdb://<site_name>/<apc_name>?<vo_name>``, for FedCloud OCCI or OpenStack connector using AppDB info (from vers. 1.6.0 and 1.8.6).
+   * ``appdb://<site_name>/<apc_name>?<vo_name>``, for FedCloud OCCI, OpenStack or EGI connectors using AppDB info (from vers. 1.6.0, 1.8.6 and 1.10.2 respectively).
+     In case of EGI connector the ``vo_name`` is not required as it will be get from auth data.
    * ``docker://<docker_image>``, for Docker images.
    * ``fbw://<fns_server>/<image-id>``, for FogBow images.
    * ``lin://linode/<image-id>``, for Linode images.