merge devel

IM/AppDB.py

@@ -37,7 +37,8 @@ def appdb_call(path):
         resp = requests.request("GET", AppDB.APPDB_URL + path, verify=False)
         if resp.status_code == 200:
             resp.text.replace('\n', '')
-            return xmltodict.parse(resp.text)
+            res = xmltodict.parse(resp.text)
+            return res['appdb:appdb'] if 'appdb:appdb' in res else res
         else:
             return None
 
@@ -48,7 +49,7 @@ def get_site_id(site_name, stype="occi"):
         """
         data = AppDB.appdb_call('/rest/1.0/sites')
         if data:
-            for site in data['appdb:appdb']['appdb:site']:
+            for site in data['appdb:site']:
                 if site_name.lower() == site['@name'].lower() and site['@infrastructure'] == "Production":
                     if isinstance(site['site:service'], list):
                         services = site['site:service']
@@ -71,13 +72,13 @@ def get_site_url(site_id, stype="occi"):
         site_url = None
         if data:
             if stype == "openstack":
-                if 'provider:url' in data['appdb:appdb']['virtualization:provider']:
-                    site_url = data['appdb:appdb']['virtualization:provider']['provider:url']
+                if 'provider:url' in data['virtualization:provider']:
+                    site_url = data['virtualization:provider']['provider:url']
                     url = urlparse(site_url)
                     site_url = "%s://%s" % url[0:2]
             else:
-                if 'provider:endpoint_url' in data['appdb:appdb']['virtualization:provider']:
-                    site_url = data['appdb:appdb']['virtualization:provider']["provider:endpoint_url"]
+                if 'provider:endpoint_url' in data['virtualization:provider']:
+                    site_url = data['virtualization:provider']["provider:endpoint_url"]
 
         return site_url
 
@@ -89,8 +90,8 @@ def get_image_id(site_id, image_name, vo_name):
         images = []
         data = AppDB.appdb_call('/rest/1.0/va_providers/%s' % site_id)
         if data:
-            if 'provider:image' in data['appdb:appdb']['virtualization:provider']:
-                for image in data['appdb:appdb']['virtualization:provider']['provider:image']:
+            if 'provider:image' in data['virtualization:provider']:
+                for image in data['virtualization:provider']['provider:image']:
                     if (image['@appcname'] == image_name and (not vo_name or image['@voname'] == vo_name)):
                         image_basename = os.path.basename(image['@va_provider_image_id'])
                         images.append((image_basename, image['@vmiversion']))
@@ -121,7 +122,7 @@ def _get_site_name(site_host, stype="occi"):
         """
         data = AppDB.appdb_call('/rest/1.0/sites')
         if data:
-            for site in data['appdb:appdb']['appdb:site']:
+            for site in data['appdb:site']:
                 if site['@infrastructure'] == "Production" and 'site:service' in site:
                     if isinstance(site['site:service'], list):
                         services = site['site:service']
@@ -193,8 +194,8 @@ def get_image_id_from_uri(site_id, image_mp_uri):
 
         data = AppDB.appdb_call('/rest/1.0/va_providers/%s' % site_id)
         if data:
-            if 'provider:image' in data['appdb:appdb']['virtualization:provider']:
-                for image in data['appdb:appdb']['virtualization:provider']['provider:image']:
+            if 'provider:image' in data['virtualization:provider']:
+                for image in data['virtualization:provider']['provider:image']:
                     if image['@mp_uri'] == image_mp_uri:
                         image_basename = os.path.basename(image['@va_provider_image_id'])
                         parts = image_basename.split("#")

IM/CloudInfo.py

@@ -127,7 +127,7 @@ def get_cloud_list(auth_data):
                             cloud_item.extra['tenant'] = auth['tenant']
                 elif 'type' in auth and auth['type'] == "EGI":
                     if 'vo' in auth and auth['vo']:
-                        cloud_item.extra["vo"] = 'vo'
+                        cloud_item.extra["vo"] = auth['vo']
 
                 res.append(cloud_item)
 

IM/connectors/EGI.py

@@ -122,6 +122,9 @@ def concrete_system(self, radl_system, str_url, auth_data):
                 site_url = AppDB.get_site_url(site_id, stype="openstack")
                 if site_url:
                     site_host = urlparse(site_url)[1].split(':')[0]
+            elif not url[2]:
+                # in case if appdb url without setting the site
+                src_host = self.cloud.server
 
             if ((protocol == "ost" and site_host == src_host) or
                     (protocol == "appdb" and src_host == self.cloud.server)):

changelog

@@ -697,3 +697,7 @@ IM 1.11.0:
     * Improve TOSCA get_attribute function.
     * Enable to get Ansible task output at TOSCA output value.
     * Support for specifying the remote cidr in the public security group.
+    * Fix error in EGI connector using appdb image url without setting site name.
+    * Fix error getting project ids from AppDB.
+
+

doc/source/client.rst

@@ -196,6 +196,11 @@ The :program:`im_client` is called like this::
 Authorization File
 ------------------
 
+To access the IM service an authenticatio file must be created.
+It must have one line per authentication element. **It must have at least
+one line with the authentication data for the IM service** and another
+one for the Cloud/s provider/s the user want to access.
+
 The authorization file stores in plain text the credentials to access the
 cloud providers, the IM service and the VMRC service. Each line of the file
 is composed by pairs of key and value separated by semicolon, and refers to a

doc/source/gstarted.rst

@@ -38,7 +38,7 @@ In this first examples we will use the IM-client tool to create, manage and fina
 Authentication file
 ^^^^^^^^^^^^^^^^^^^
 To access the IM service an authenticatio file must be created. It must have one line per authentication element.
-It must have at least one line with the authentication data for the IM service and another one for the Cloud/s
+**It must have at least one line with the authentication data for the IM service** and another one for the Cloud/s
 provider/s the user want to access.
 
 An example to access an OpenNebula and/or an OpenStack site::
@@ -69,12 +69,12 @@ specify an existing image on the cluod provider where VM will be launched. O.S.
 Cloud providers:
 
    * **one://<server>:<port>/<image-id>**, for OpenNebula;
-   * **ost://<server>/<ami-id>**, for OpenStack;
+   * **ost://<server>/<ami-id>**, for OpenStack or EGI;
    * **aws://<region>/<ami-id>**, for Amazon Web Service;
    * **gce://<region>/<image-id>**, for Google Cloud;
    * **azr://<publisher>/<offer>/<sku>/<version>**, for Microsoft Azure; and
    * **<fedcloud_endpoint_url>/<image_id>**, for FedCloud OCCI connector.
-   * **appdb://<site_name>/<apc_name>?<vo_name>**, for FedCloud OCCI connector using AppDB info (from ver. 1.6.0).
+   * **appdb://<site_name>/<apc_name>?<vo_name>**, for FedCloud OCCI, EGI or OpenStack connector using AppDB info.
    * **docker://<docker_image>**, for Docker images.
    * **fbw://<fns_server>/<image-id>**, for FogBow images.
    * **lin://linode/<image-id>**, for Linode images.
@@ -96,11 +96,11 @@ should be like that::
 
     topology_template:
     
-    node_templates:
+      node_templates:
     
-      simple_node:
-        type: tosca.nodes.indigo.Compute
-        capabilities:
+        simple_node:
+          type: tosca.nodes.indigo.Compute
+          capabilities:
             endpoint:
               properties:
                 network_name: PUBLIC
@@ -112,7 +112,7 @@ should be like that::
               properties:
                 image: one://someserver.com/123
 
-    outputs:
+      outputs:
         node_ip:
           value: { get_attribute: [ simple_node, public_address, 0 ] }
         node_creds:

doc/source/manual.rst

@@ -73,26 +73,26 @@ Finally, check the next values in the Ansible configuration file
 Optional Packages
 -----------------
 
-* `The Bottle framework<http://bottlepy.org/>`_ is used for the REST API. 
-   It is typically available as the 'python-bottle' package.
+* `The Bottle framework <http://bottlepy.org/>`_ is used for the REST API. 
+  It is typically available as the 'python-bottle' package.
 * `The CherryPy Web framework <http://www.cherrypy.org/>`_, is needed for the REST API. 
-   It is typically available as the 'python-cherrypy' or 'python-cherrypy3' package.
-   In newer versions (9.0 and later) the functionality has been moved `the cheroot
-   library<https://github.com/cherrypy/cheroot>`_ it can be installed using pip.
+  It is typically available as the 'python-cherrypy' or 'python-cherrypy3' package.
+  In newer versions (9.0 and later) the functionality has been moved `the cheroot
+  library <https://github.com/cherrypy/cheroot>`_ it can be installed using pip.
 * `apache-libcloud <http://libcloud.apache.org/>`_ 3.0 or later is used in the
   LibCloud, OpenStack, EGI and GCE connectors.
 * `boto <http://boto.readthedocs.org>`_ 2.29.0 or later is used as interface to
   Amazon EC2. It is available as package named ``python-boto`` in Debian based
   distributions. It can also be downloaded from `boto GitHub repository <https://github.com/boto/boto>`_.
   Download the file and copy the boto subdirectory into the IM install path.
-* pyOpenSSL are needed if needed to secure the REST API
+* `pyOpenSSL <https://www.pyopenssl.org/>`_ is needed to secure the REST API
   with SSL certificates (see :confval:`REST_SSL`).
   pyOpenSSL can be installed using pip.
 * `The Python interface to MySQL <https://www.mysql.com/>`_, is needed to access MySQL server as IM data 
   backend. It is typically available as the package 'python-mysqldb' or 'MySQL-python' package. In case of
   using Python 3 use the PyMySQL package, available as the package 'python3-pymysql' on debian systems or PyMySQL
   package in pip.
-  *`The Python interface to MongoDB <https://www.mongodb.com/>`_, is needed to access MongoDB server as IM data 
+* `The Python interface to MongoDB <https://www.mongodb.com/>`_, is needed to access MongoDB server as IM data 
   backend. It is typically available as the package 'python-pymongo' package in most distributions or pymongo
   package in pip.
 * `The Azure Python SDK <https://docs.microsoft.com/es-es/azure/python-how-to-install/>`_, is needed by the Azure

test/unit/AppDB.py

@@ -34,6 +34,10 @@ def get_response(self, method, url, verify, cert=None, headers=None, data=None):
                 resp.status_code = 200
                 resp.text = """<appdb:appdb>
                                 <virtualization:provider id="8015G0" in_production="true">
+                                <provider:shares>
+                                  <vo:vo id="1" projectid="projectid1">ops</vo:vo>
+                                  <vo:vo id="2" projectid="projectid2">vo.access.egi.eu</vo:vo>
+                                </provider:shares>
                                 <provider:url>
                                 http://cloud.recas.ba.infn.it:8787/occi/?image=303d8324-69a7-4372-be24-1d68703affd7
                                 </provider:url>
@@ -121,3 +125,15 @@ def test_get_image_data(self, requests):
         site_url, image_id, _ = AppDB.get_image_data(str_url, "openstack", site='cloud.recas.ba.infn.it')
         self.assertEqual(site_url, "https://cloud.recas.ba.infn.it:5000")
         self.assertEqual(image_id, "image_id2")
+
+        str_url = "appdb://egi.ubuntu.16.04"
+        site_url, image_id, _ = AppDB.get_image_data(str_url, "openstack", vo="fedcloud.egi.eu",
+                                                     site='cloud.recas.ba.infn.it')
+        self.assertEqual(site_url, "https://cloud.recas.ba.infn.it:5000")
+        self.assertEqual(image_id, "image_id2")
+
+    @patch('requests.request')
+    def test_get_project_ids(self, requests):
+        requests.side_effect = self.get_response
+        projects = AppDB.get_project_ids('8015G0')
+        self.assertEqual(projects, {'ops': 'projectid1', 'vo.access.egi.eu': 'projectid2'})

test/unit/connectors/EGI.py

@@ -103,6 +103,12 @@ def test_10_concrete(self, appdb, get_driver):
         self.assertEqual(concrete[0].getValue("instance_type"), "small")
         self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
 
+        radl_system.setValue('disk.0.image.url', 'appdb://image_apc_name')
+        concrete = egi_cloud.concreteSystem(radl_system, auth)
+        self.assertEqual(len(concrete), 1)
+        self.assertEqual(concrete[0].getValue("instance_type"), "small")
+        self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
+
     def create_node(self, **kwargs):
         """
         Create VMs returning error only first time
@@ -133,7 +139,7 @@ def test_20_launch(self, appdb, get_image_data, save_data, get_driver):
             net_interface.1.connection = 'net1' and
             net_interface.0.connection = 'net2' and
             disk.0.os.name = 'linux' and
-            disk.0.image.url = 'ost://server.com/ami-id' and
+            disk.0.image.url = 'appdb://apc_image_name' and
             disk.0.os.credentials.username = 'user' and
             disk.1.size=1GB and
             disk.1.device='hdb' and
@@ -152,6 +158,7 @@ def test_20_launch(self, appdb, get_image_data, save_data, get_driver):
         driver = MagicMock()
         get_driver.return_value = driver
 
+        get_image_data.return_value = "", "imageid", ""
         appdb.get_site_id.return_value = "site1"
         appdb.get_site_url.return_value = "https://site.com:5000/v3"
         appdb.get_project_ids.return_value = {"vo.access.egi.eu": "projectid"}