Merge pull request #619 from grycap/devel

Devel

IM/connectors/Azure.py

@@ -26,6 +26,7 @@
 try:
     from azure.mgmt.resource import ResourceManagementClient
     from azure.mgmt.storage import StorageManagementClient
+    from azure.storage.blob import BlockBlobService
     from azure.mgmt.compute import ComputeManagementClient
     from azure.mgmt.network import NetworkManagementClient
     from azure.mgmt.dns import DnsManagementClient
@@ -405,11 +406,12 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc
         if system.getValue('availability_zone'):
             location = system.getValue('availability_zone')
 
-        # Allways use the new credentials
+        # Always use the new credentials
         system.updateNewCredentialValues()
         user_credentials = system.getCredentials()
 
         os_disk_name = "osdisk-" + str(uuid.uuid1())
+        disks = [vm_name + os_disk_name + ".vhd"]
 
         vm = {
             'location': location,
@@ -443,10 +445,23 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc
             },
         }
 
+        tags = {}
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                tags[key] = value
+
+        if tags:
+            vm['tags'] = tags
+
         cont = 1
         data_disks = []
         while system.getValue("disk." + str(cont) + ".size"):
             disk_size = system.getFeature("disk." + str(cont) + ".size").getValue('G')
+            disks.append("{}disk{}.vhd".format(vm_name, cont))
             self.log_info("Adding a %s GB disk." % disk_size)
             data_disks.append({
                 'name': '%s_disk_%d' % (vm_name, cont),
@@ -463,7 +478,7 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc
         if data_disks:
             vm['storage_profile']['data_disks'] = data_disks
 
-        return vm
+        return vm, disks
 
     def create_nets(self, radl, credentials, subscription_id, group_name):
         network_client = NetworkManagementClient(credentials, subscription_id)
@@ -529,17 +544,31 @@ def create_vms(self, inf, radl, requested_radl, num_vm, location, storage_accoun
             group_name = "rg-%s" % (vm_name)
 
             try:
+                tags = {}
+                if radl.systems[0].getValue('instance_tags'):
+                    keypairs = radl.systems[0].getValue('instance_tags').split(",")
+                    for keypair in keypairs:
+                        parts = keypair.split("=")
+                        key = parts[0].strip()
+                        value = parts[1].strip()
+                        tags[key] = value
+
+                args = {'location': location}
+                if tags:
+                    args['tags'] = tags
+
                 # Create resource group for the VM
-                resource_client.resource_groups.create_or_update(group_name, {'location': location})
+                resource_client.resource_groups.create_or_update(group_name, args)
 
                 vm = VirtualMachine(inf, group_name + '/' + vm_name, self.cloud, radl, requested_radl, self)
                 vm.info.systems[0].setValue('instance_id', group_name + '/' + vm_name)
 
                 nics = self.create_nics(radl, credentials, subscription_id, group_name, subnets)
 
                 instance_type = self.get_instance_type(radl.systems[0], credentials, subscription_id)
-                vm_parameters = self.get_azure_vm_create_json(storage_account_name, vm_name,
-                                                              nics, radl, instance_type)
+                vm_parameters, disks = self.get_azure_vm_create_json(storage_account_name, vm_name,
+                                                                     nics, radl, instance_type)
+                vm.disks = disks
 
                 compute_client = ComputeManagementClient(credentials, subscription_id)
                 async_vm_creation = compute_client.virtual_machines.create_or_update(group_name,
@@ -556,11 +585,21 @@ def create_vms(self, inf, radl, requested_radl, num_vm, location, storage_accoun
                 # Delete Resource group and everything in it
                 if group_name:
                     self.delete_resource_group(group_name, resource_client)
+                    self.delete_vm_disks(vm, credentials, subscription_id)
 
             i += 1
 
         return vms
 
+    @staticmethod
+    def get_storage_account_name(inf_id):
+        # Storage account name must be between 3 and 24 characters in length and use
+        # numbers and lower-case letters only
+        storage_account_name = "s%s" % inf_id
+        storage_account_name = storage_account_name.replace("-", "")
+        storage_account_name = storage_account_name[:24]
+        return storage_account_name
+
     def launch(self, inf, radl, requested_radl, num_vm, auth_data):
         location = self.DEFAULT_LOCATION
         if radl.systems[0].getValue('availability_zone'):
@@ -572,11 +611,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
 
         resource_client = ResourceManagementClient(credentials, subscription_id)
 
-        # Storage account name must be between 3 and 24 characters in length and use
-        # numbers and lower-case letters only
-        storage_account_name = "s%s" % inf.id
-        storage_account_name = storage_account_name.replace("-", "")
-        storage_account_name = storage_account_name[:24]
+        storage_account_name = self.get_storage_account_name(inf.id)
 
         with inf._lock:
             # Create resource group for the Infrastructure if it does not exists
@@ -646,6 +681,14 @@ def updateVMInfo(self, vm, auth_data):
             compute_client = ComputeManagementClient(credentials, subscription_id)
             # Get one the virtual machine by name
             virtual_machine = compute_client.virtual_machines.get(group_name, vm_name, expand='instanceView')
+
+            if virtual_machine.storage_profile.data_disks:
+                for data_disk in virtual_machine.storage_profile.data_disks:
+                    self.log_debug(data_disk.vhd.name)
+
+            if virtual_machine.storage_profile.os_disk:
+                self.log_debug(virtual_machine.storage_profile.os_disk.name)
+
         except Exception as ex:
             self.log_warn("The VM does not exists.")
             # check if the RG still exists
@@ -778,6 +821,8 @@ def finalize(self, vm, last, auth_data):
                         return False, "Error terminating the VM: %s" % msg
                 else:
                     self.log_info("RG: %s does not exist. Do not remove." % "rg-%s" % vm.inf.id)
+            else:
+                self.delete_vm_disks(vm, credentials, subscription_id)
 
         except Exception as ex:
             self.log_exception("Error terminating the VM")
@@ -865,3 +910,29 @@ def delete_resource_group(self, group_name, resource_client, max_retries=3):
             self.log_info("Resource group %s successfully deleted." % group_name)
 
         return deleted, msg
+
+    def delete_vm_disks(self, vm, credentials, subscription_id):
+        try:
+            if "disks" in vm.__dict__.keys():
+                storage_account_name = self.get_storage_account_name(vm.inf.id)
+
+                storage_client = StorageManagementClient(credentials, subscription_id)
+                keys = storage_client.storage_accounts.list_keys("rg-%s" % vm.inf.id, storage_account_name)
+
+                key = None
+                for key in keys.keys:
+                    break
+                if not key:
+                    self.log_error("Error deleting VM disks: No key found.")
+                    return (False, "Error deleting VM disks: No key found.")
+
+                block_blob_service = BlockBlobService(account_name=storage_account_name, account_key=key.value)
+
+                for disk in vm.disks:
+                    self.log_debug("Deleting disk: %s" % disk)
+                    block_blob_service.delete_blob("vhds", disk)
+        except Exception as ex:
+            self.log_exception("Error deleting VM disks")
+            return (False, "Error deleting VM disks" + str(ex))
+
+        return True, ""

IM/connectors/EC2.py

@@ -596,6 +596,15 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                     res.append((False, "Error managing the keypair."))
                 return res
 
+            tags = {}
+            if system.getValue('instance_tags'):
+                keypairs = system.getValue('instance_tags').split(",")
+                for keypair in keypairs:
+                    parts = keypair.split("=")
+                    key = parts[0].strip()
+                    value = parts[1].strip()
+                    tags[key] = value
+
             all_failed = True
 
             i = 0
@@ -711,6 +720,8 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                             if len(reservation.instances) == 1:
                                 instance = reservation.instances[0]
                                 instance.add_tag("IM-USER", im_username)
+                                for key, value in tags.items():
+                                    instance.add_tag(key, value)
                                 ec2_vm_id = region_name + ";" + instance.id
 
                                 self.log_debug("RADL:")
@@ -796,7 +807,7 @@ def attach_volumes(self, instance, vm):
         """
         try:
             if instance.state == 'running' and "volumes" not in vm.__dict__.keys():
-                # Flag to se that this VM has created (or is creating) the
+                # Flag to set that this VM has created (or is creating) the
                 # volumes
                 vm.volumes = True
                 conn = instance.connection

IM/connectors/GCE.py

@@ -456,6 +456,16 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                 'external_ip': None,
                 'location': region}
 
+        tags = {}
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                tags[key] = value
+            args['ex_metadata'] = tags
+
         # include the SSH_KEYS
         username = system.getValue('disk.0.os.credentials.username')
         private = system.getValue('disk.0.os.credentials.private_key')

IM/connectors/OpenNebula.py

@@ -646,6 +646,18 @@ def getONETemplate(self, radl, sgs, auth_data):
             %s
         ''' % (name, cpu, cpu, memory, arch, disks, ConfigOpenNebula.TEMPLATE_OTHER)
 
+        user_template = ""
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                user_template += '%s = "%s", ' % (key, value)
+
+        if user_template:
+            res += "\nUSER_TEMPLATE = [%s]\n" % user_template[:-2]
+
         res += self.get_networks_template(radl, sgs, auth_data)
 
         # include the SSH_KEYS

IM/connectors/OpenStack.py

@@ -534,6 +534,16 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
                 'ex_security_groups': sgs,
                 'name': "%s-%s" % (name, int(time.time() * 100))}
 
+        tags = {}
+        if system.getValue('instance_tags'):
+            keypairs = system.getValue('instance_tags').split(",")
+            for keypair in keypairs:
+                parts = keypair.split("=")
+                key = parts[0].strip()
+                value = parts[1].strip()
+                tags[key] = value
+            args['ex_metadata'] = tags
+
         keypair = None
         keypair_name = None
         keypair_created = False

README.md

@@ -168,7 +168,7 @@ $ yum localinstall IM-*.rpm RADL-*.rpm
 Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip:
 
 ```sh
-$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns 
+$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage 
 ```
 
 #### 1.3.4 From Deb package (Tested with Ubuntu 14.04 and 16.04)

ansible_install.yaml

@@ -76,7 +76,7 @@
       ignore_errors: yes
 
     - name: Install pip libraries
-      pip: executable=pip name=pyOpenSSL,MySQL-python,msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource,azure-mgmt-dns
+      pip: executable=pip name=pyOpenSSL,MySQL-python,msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource,azure-mgmt-dns,azure-storage
 
     - name: Install IM dependencies for CentOS 6
       pip: name=pysqlite version=2.7.0

changelog

@@ -414,3 +414,5 @@ IM 1.7.1:
 IM 1.7.2:
     * Fix Error setting the INF_CACHE_TIME conf variable time.
     * Add support to availability_zone in tosca.policies.Placement.
+    * Enable to set instance_tags in connectors.
+    * Fix error in Azure conn: VM disks are not deleted when VM is finalized, only when Infrastructure is destroyed.

doc/source/manual.rst

@@ -125,7 +125,7 @@ Then install the downloaded RPMs::
 
 Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip::
 
-	$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns
+	$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage
 
 From Deb package (Tested with Ubuntu 14.04 and 16.04)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

docker-devel/Dockerfile

@@ -7,14 +7,15 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv
 
 EXPOSE 8899 8800
 
-# Install pip optional libraries
-RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns
 # Install im - '$BRANCH' branch
 RUN cd tmp \
  && git clone -b $BRANCH https://github.com/grycap/im.git \
  && cd im \
  && pip install /tmp/im
 
+# Install pip optional libraries
+RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage
+
 # Set the VM_NUM_USE_CTXT_DIST to 3 for the tests
 RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/im.cfg
 

docker-py3/Dockerfile

@@ -12,7 +12,7 @@ RUN pip3 install setuptools pip --upgrade -I
 RUN pip3 install pyOpenSSL --upgrade -I
 
 # Install pip optional libraries
-RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns cheroot xmltodict
+RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage cheroot xmltodict
 
 # Install IM
 RUN apt-get update && apt-get install --no-install-recommends -y gcc libssl-dev libffi-dev libsqlite3-dev && \

docker/Dockerfile

@@ -11,9 +11,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y python-dbg pyth
 RUN pip install setuptools pip --upgrade -I
 RUN pip install pyOpenSSL --upgrade -I
 
-# Install pip optional libraries
-RUN pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns cheroot xmltodict
-
 # Install IM
 RUN apt-get update && apt-get install --no-install-recommends -y gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev libmysqlclient20 && \
      pip install pycrypto && \
@@ -22,6 +19,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y gcc libmysqld-d
      apt-get purge -y gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev python-dev python-pip && \
      apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/
 
+# Install pip optional libraries
+RUN pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage cheroot xmltodict
+
 # Force requests to be version 2.11.1 to avoid SSL ca errors with proxy files
 RUN pip install requests==2.11.1
 

test/unit/connectors/Azure.py

@@ -153,6 +153,7 @@ def test_20_launch(self, save_data, credentials, network_client, compute_client,
             cpu.arch='x86_64' and
             cpu.count>=1 and
             memory.size>=512m and
+            instance_tags = 'key=value,key1=value2' and
             net_interface.0.connection = 'net1' and
             net_interface.0.dns_name = 'test' and
             net_interface.1.connection = 'net2' and
@@ -394,9 +395,11 @@ def test_55_alter(self, credentials, network_client, compute_client, storage_cli
         self.assertTrue(success, msg="ERROR: modifying VM info.")
         self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
 
+    @patch('IM.connectors.Azure.BlockBlobService')
+    @patch('IM.connectors.Azure.StorageManagementClient')
     @patch('IM.connectors.Azure.ResourceManagementClient')
     @patch('IM.connectors.Azure.UserPassCredentials')
-    def test_60_finalize(self, credentials, resource_client):
+    def test_60_finalize(self, credentials, resource_client, storage_client, blob):
         auth = Authentication([{'id': 'azure', 'type': 'Azure', 'subscription_id': 'subscription_id',
                                 'username': 'user', 'password': 'password'}])
         azure_cloud = self.get_azure_cloud()
@@ -411,9 +414,17 @@ def test_60_finalize(self, credentials, resource_client):
             )"""
         radl = radl_parse.parse_radl(radl_data)
 
+        key = MagicMock()
+        key.keys = [MagicMock()]
+        sclient = MagicMock()
+        storage_client.return_value = sclient
+        sclient.storage_accounts.list_keys.return_value = key
+
         inf = MagicMock()
         vm = VirtualMachine(inf, "rg0/vm0", azure_cloud.cloud, radl, radl, azure_cloud, 1)
+        vm.disks = ["disk1"]
 
+        success, _ = azure_cloud.finalize(vm, False, auth)
         success, _ = azure_cloud.finalize(vm, True, auth)
 
         self.assertTrue(success, msg="ERROR: finalizing VM info.")

test/unit/connectors/EC2.py

@@ -126,6 +126,7 @@ def test_20_launch(self, save_data, blockdevicemapping, VPCConnection, get_regio
             cpu.arch='x86_64' and
             cpu.count>=1 and
             memory.size>=512m and
+            instance_tags = 'key=value,key1=value2' and
             net_interface.0.connection = 'net1' and
             net_interface.0.dns_name = 'test' and
             net_interface.1.connection = 'net2' and