diff --git a/IM/connectors/Azure.py b/IM/connectors/Azure.py index 4bf007f1e..127a9d33a 100644 --- a/IM/connectors/Azure.py +++ b/IM/connectors/Azure.py @@ -26,6 +26,7 @@ try: from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.storage import StorageManagementClient + from azure.storage.blob import BlockBlobService from azure.mgmt.compute import ComputeManagementClient from azure.mgmt.network import NetworkManagementClient from azure.mgmt.dns import DnsManagementClient @@ -405,11 +406,12 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc if system.getValue('availability_zone'): location = system.getValue('availability_zone') - # Allways use the new credentials + # Always use the new credentials system.updateNewCredentialValues() user_credentials = system.getCredentials() os_disk_name = "osdisk-" + str(uuid.uuid1()) + disks = [vm_name + os_disk_name + ".vhd"] vm = { 'location': location, @@ -443,10 +445,23 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc }, } + tags = {} + if system.getValue('instance_tags'): + keypairs = system.getValue('instance_tags').split(",") + for keypair in keypairs: + parts = keypair.split("=") + key = parts[0].strip() + value = parts[1].strip() + tags[key] = value + + if tags: + vm['tags'] = tags + cont = 1 data_disks = [] while system.getValue("disk." + str(cont) + ".size"): disk_size = system.getFeature("disk." + str(cont) + ".size").getValue('G') + disks.append("{}disk{}.vhd".format(vm_name, cont)) self.log_info("Adding a %s GB disk." % disk_size) data_disks.append({ 'name': '%s_disk_%d' % (vm_name, cont), @@ -463,7 +478,7 @@ def get_azure_vm_create_json(self, storage_account, vm_name, nics, radl, instanc if data_disks: vm['storage_profile']['data_disks'] = data_disks - return vm + return vm, disks def create_nets(self, radl, credentials, subscription_id, group_name): network_client = NetworkManagementClient(credentials, subscription_id) @@ -529,8 +544,21 @@ def create_vms(self, inf, radl, requested_radl, num_vm, location, storage_accoun group_name = "rg-%s" % (vm_name) try: + tags = {} + if radl.systems[0].getValue('instance_tags'): + keypairs = radl.systems[0].getValue('instance_tags').split(",") + for keypair in keypairs: + parts = keypair.split("=") + key = parts[0].strip() + value = parts[1].strip() + tags[key] = value + + args = {'location': location} + if tags: + args['tags'] = tags + # Create resource group for the VM - resource_client.resource_groups.create_or_update(group_name, {'location': location}) + resource_client.resource_groups.create_or_update(group_name, args) vm = VirtualMachine(inf, group_name + '/' + vm_name, self.cloud, radl, requested_radl, self) vm.info.systems[0].setValue('instance_id', group_name + '/' + vm_name) @@ -538,8 +566,9 @@ def create_vms(self, inf, radl, requested_radl, num_vm, location, storage_accoun nics = self.create_nics(radl, credentials, subscription_id, group_name, subnets) instance_type = self.get_instance_type(radl.systems[0], credentials, subscription_id) - vm_parameters = self.get_azure_vm_create_json(storage_account_name, vm_name, - nics, radl, instance_type) + vm_parameters, disks = self.get_azure_vm_create_json(storage_account_name, vm_name, + nics, radl, instance_type) + vm.disks = disks compute_client = ComputeManagementClient(credentials, subscription_id) async_vm_creation = compute_client.virtual_machines.create_or_update(group_name, @@ -556,11 +585,21 @@ def create_vms(self, inf, radl, requested_radl, num_vm, location, storage_accoun # Delete Resource group and everything in it if group_name: self.delete_resource_group(group_name, resource_client) + self.delete_vm_disks(vm, credentials, subscription_id) i += 1 return vms + @staticmethod + def get_storage_account_name(inf_id): + # Storage account name must be between 3 and 24 characters in length and use + # numbers and lower-case letters only + storage_account_name = "s%s" % inf_id + storage_account_name = storage_account_name.replace("-", "") + storage_account_name = storage_account_name[:24] + return storage_account_name + def launch(self, inf, radl, requested_radl, num_vm, auth_data): location = self.DEFAULT_LOCATION if radl.systems[0].getValue('availability_zone'): @@ -572,11 +611,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): resource_client = ResourceManagementClient(credentials, subscription_id) - # Storage account name must be between 3 and 24 characters in length and use - # numbers and lower-case letters only - storage_account_name = "s%s" % inf.id - storage_account_name = storage_account_name.replace("-", "") - storage_account_name = storage_account_name[:24] + storage_account_name = self.get_storage_account_name(inf.id) with inf._lock: # Create resource group for the Infrastructure if it does not exists @@ -646,6 +681,14 @@ def updateVMInfo(self, vm, auth_data): compute_client = ComputeManagementClient(credentials, subscription_id) # Get one the virtual machine by name virtual_machine = compute_client.virtual_machines.get(group_name, vm_name, expand='instanceView') + + if virtual_machine.storage_profile.data_disks: + for data_disk in virtual_machine.storage_profile.data_disks: + self.log_debug(data_disk.vhd.name) + + if virtual_machine.storage_profile.os_disk: + self.log_debug(virtual_machine.storage_profile.os_disk.name) + except Exception as ex: self.log_warn("The VM does not exists.") # check if the RG still exists @@ -778,6 +821,8 @@ def finalize(self, vm, last, auth_data): return False, "Error terminating the VM: %s" % msg else: self.log_info("RG: %s does not exist. Do not remove." % "rg-%s" % vm.inf.id) + else: + self.delete_vm_disks(vm, credentials, subscription_id) except Exception as ex: self.log_exception("Error terminating the VM") @@ -865,3 +910,29 @@ def delete_resource_group(self, group_name, resource_client, max_retries=3): self.log_info("Resource group %s successfully deleted." % group_name) return deleted, msg + + def delete_vm_disks(self, vm, credentials, subscription_id): + try: + if "disks" in vm.__dict__.keys(): + storage_account_name = self.get_storage_account_name(vm.inf.id) + + storage_client = StorageManagementClient(credentials, subscription_id) + keys = storage_client.storage_accounts.list_keys("rg-%s" % vm.inf.id, storage_account_name) + + key = None + for key in keys.keys: + break + if not key: + self.log_error("Error deleting VM disks: No key found.") + return (False, "Error deleting VM disks: No key found.") + + block_blob_service = BlockBlobService(account_name=storage_account_name, account_key=key.value) + + for disk in vm.disks: + self.log_debug("Deleting disk: %s" % disk) + block_blob_service.delete_blob("vhds", disk) + except Exception as ex: + self.log_exception("Error deleting VM disks") + return (False, "Error deleting VM disks" + str(ex)) + + return True, "" diff --git a/IM/connectors/EC2.py b/IM/connectors/EC2.py index 2f3e3d09e..a62a60899 100644 --- a/IM/connectors/EC2.py +++ b/IM/connectors/EC2.py @@ -596,6 +596,15 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): res.append((False, "Error managing the keypair.")) return res + tags = {} + if system.getValue('instance_tags'): + keypairs = system.getValue('instance_tags').split(",") + for keypair in keypairs: + parts = keypair.split("=") + key = parts[0].strip() + value = parts[1].strip() + tags[key] = value + all_failed = True i = 0 @@ -711,6 +720,8 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): if len(reservation.instances) == 1: instance = reservation.instances[0] instance.add_tag("IM-USER", im_username) + for key, value in tags.items(): + instance.add_tag(key, value) ec2_vm_id = region_name + ";" + instance.id self.log_debug("RADL:") @@ -796,7 +807,7 @@ def attach_volumes(self, instance, vm): """ try: if instance.state == 'running' and "volumes" not in vm.__dict__.keys(): - # Flag to se that this VM has created (or is creating) the + # Flag to set that this VM has created (or is creating) the # volumes vm.volumes = True conn = instance.connection diff --git a/IM/connectors/GCE.py b/IM/connectors/GCE.py index 3d82a490c..eeb38664f 100644 --- a/IM/connectors/GCE.py +++ b/IM/connectors/GCE.py @@ -456,6 +456,16 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): 'external_ip': None, 'location': region} + tags = {} + if system.getValue('instance_tags'): + keypairs = system.getValue('instance_tags').split(",") + for keypair in keypairs: + parts = keypair.split("=") + key = parts[0].strip() + value = parts[1].strip() + tags[key] = value + args['ex_metadata'] = tags + # include the SSH_KEYS username = system.getValue('disk.0.os.credentials.username') private = system.getValue('disk.0.os.credentials.private_key') diff --git a/IM/connectors/OpenNebula.py b/IM/connectors/OpenNebula.py index 94d203c48..0417cc879 100644 --- a/IM/connectors/OpenNebula.py +++ b/IM/connectors/OpenNebula.py @@ -646,6 +646,18 @@ def getONETemplate(self, radl, sgs, auth_data): %s ''' % (name, cpu, cpu, memory, arch, disks, ConfigOpenNebula.TEMPLATE_OTHER) + user_template = "" + if system.getValue('instance_tags'): + keypairs = system.getValue('instance_tags').split(",") + for keypair in keypairs: + parts = keypair.split("=") + key = parts[0].strip() + value = parts[1].strip() + user_template += '%s = "%s", ' % (key, value) + + if user_template: + res += "\nUSER_TEMPLATE = [%s]\n" % user_template[:-2] + res += self.get_networks_template(radl, sgs, auth_data) # include the SSH_KEYS diff --git a/IM/connectors/OpenStack.py b/IM/connectors/OpenStack.py index 5517d3781..8242c63b8 100644 --- a/IM/connectors/OpenStack.py +++ b/IM/connectors/OpenStack.py @@ -534,6 +534,16 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data): 'ex_security_groups': sgs, 'name': "%s-%s" % (name, int(time.time() * 100))} + tags = {} + if system.getValue('instance_tags'): + keypairs = system.getValue('instance_tags').split(",") + for keypair in keypairs: + parts = keypair.split("=") + key = parts[0].strip() + value = parts[1].strip() + tags[key] = value + args['ex_metadata'] = tags + keypair = None keypair_name = None keypair_created = False diff --git a/README.md b/README.md index bb5e173f9..b1d612ea2 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ $ yum localinstall IM-*.rpm RADL-*.rpm Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip: ```sh -$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns +$ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage ``` #### 1.3.4 From Deb package (Tested with Ubuntu 14.04 and 16.04) diff --git a/ansible_install.yaml b/ansible_install.yaml index b98d1cde6..04b8f81e9 100644 --- a/ansible_install.yaml +++ b/ansible_install.yaml @@ -76,7 +76,7 @@ ignore_errors: yes - name: Install pip libraries - pip: executable=pip name=pyOpenSSL,MySQL-python,msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource,azure-mgmt-dns + pip: executable=pip name=pyOpenSSL,MySQL-python,msrest,msrestazure,azure-common,azure-mgmt-storage,azure-mgmt-compute,azure-mgmt-network,azure-mgmt-resource,azure-mgmt-dns,azure-storage - name: Install IM dependencies for CentOS 6 pip: name=pysqlite version=2.7.0 diff --git a/changelog b/changelog index 00a42e4ca..59bfa32da 100644 --- a/changelog +++ b/changelog @@ -414,3 +414,5 @@ IM 1.7.1: IM 1.7.2: * Fix Error setting the INF_CACHE_TIME conf variable time. * Add support to availability_zone in tosca.policies.Placement. + * Enable to set instance_tags in connectors. + * Fix error in Azure conn: VM disks are not deleted when VM is finalized, only when Infrastructure is destroyed. diff --git a/doc/source/manual.rst b/doc/source/manual.rst index 795f80290..226236f34 100644 --- a/doc/source/manual.rst +++ b/doc/source/manual.rst @@ -125,7 +125,7 @@ Then install the downloaded RPMs:: Azure python SDK is not available in CentOS. So if you need the Azure plugin you have to manually install them using pip:: - $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns + $ pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage From Deb package (Tested with Ubuntu 14.04 and 16.04) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docker-devel/Dockerfile b/docker-devel/Dockerfile index e471de559..0b4958ab7 100644 --- a/docker-devel/Dockerfile +++ b/docker-devel/Dockerfile @@ -7,14 +7,15 @@ LABEL description="Container image to run the IM service. (http://www.grycap.upv EXPOSE 8899 8800 -# Install pip optional libraries -RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns # Install im - '$BRANCH' branch RUN cd tmp \ && git clone -b $BRANCH https://github.com/grycap/im.git \ && cd im \ && pip install /tmp/im +# Install pip optional libraries +RUN pip install MySQL-python msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage + # Set the VM_NUM_USE_CTXT_DIST to 3 for the tests RUN sed -i -e 's/VM_NUM_USE_CTXT_DIST = 30/VM_NUM_USE_CTXT_DIST = 3/g' /etc/im/im.cfg diff --git a/docker-py3/Dockerfile b/docker-py3/Dockerfile index abf889cad..7f0a6627d 100644 --- a/docker-py3/Dockerfile +++ b/docker-py3/Dockerfile @@ -12,7 +12,7 @@ RUN pip3 install setuptools pip --upgrade -I RUN pip3 install pyOpenSSL --upgrade -I # Install pip optional libraries -RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns cheroot xmltodict +RUN pip3 install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage cheroot xmltodict # Install IM RUN apt-get update && apt-get install --no-install-recommends -y gcc libssl-dev libffi-dev libsqlite3-dev && \ diff --git a/docker/Dockerfile b/docker/Dockerfile index 45137bb06..5b009cc4f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,9 +11,6 @@ RUN apt-get update && apt-get install --no-install-recommends -y python-dbg pyth RUN pip install setuptools pip --upgrade -I RUN pip install pyOpenSSL --upgrade -I -# Install pip optional libraries -RUN pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns cheroot xmltodict - # Install IM RUN apt-get update && apt-get install --no-install-recommends -y gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev libmysqlclient20 && \ pip install pycrypto && \ @@ -22,6 +19,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y gcc libmysqld-d apt-get purge -y gcc libmysqld-dev libssl-dev libffi-dev libsqlite3-dev python-dev python-pip && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ +# Install pip optional libraries +RUN pip install msrest msrestazure azure-common azure-mgmt-storage azure-mgmt-compute azure-mgmt-network azure-mgmt-resource azure-mgmt-dns azure-storage cheroot xmltodict + # Force requests to be version 2.11.1 to avoid SSL ca errors with proxy files RUN pip install requests==2.11.1 diff --git a/test/unit/connectors/Azure.py b/test/unit/connectors/Azure.py index 260136084..cf31430c7 100755 --- a/test/unit/connectors/Azure.py +++ b/test/unit/connectors/Azure.py @@ -153,6 +153,7 @@ def test_20_launch(self, save_data, credentials, network_client, compute_client, cpu.arch='x86_64' and cpu.count>=1 and memory.size>=512m and + instance_tags = 'key=value,key1=value2' and net_interface.0.connection = 'net1' and net_interface.0.dns_name = 'test' and net_interface.1.connection = 'net2' and @@ -394,9 +395,11 @@ def test_55_alter(self, credentials, network_client, compute_client, storage_cli self.assertTrue(success, msg="ERROR: modifying VM info.") self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue()) + @patch('IM.connectors.Azure.BlockBlobService') + @patch('IM.connectors.Azure.StorageManagementClient') @patch('IM.connectors.Azure.ResourceManagementClient') @patch('IM.connectors.Azure.UserPassCredentials') - def test_60_finalize(self, credentials, resource_client): + def test_60_finalize(self, credentials, resource_client, storage_client, blob): auth = Authentication([{'id': 'azure', 'type': 'Azure', 'subscription_id': 'subscription_id', 'username': 'user', 'password': 'password'}]) azure_cloud = self.get_azure_cloud() @@ -411,9 +414,17 @@ def test_60_finalize(self, credentials, resource_client): )""" radl = radl_parse.parse_radl(radl_data) + key = MagicMock() + key.keys = [MagicMock()] + sclient = MagicMock() + storage_client.return_value = sclient + sclient.storage_accounts.list_keys.return_value = key + inf = MagicMock() vm = VirtualMachine(inf, "rg0/vm0", azure_cloud.cloud, radl, radl, azure_cloud, 1) + vm.disks = ["disk1"] + success, _ = azure_cloud.finalize(vm, False, auth) success, _ = azure_cloud.finalize(vm, True, auth) self.assertTrue(success, msg="ERROR: finalizing VM info.") diff --git a/test/unit/connectors/EC2.py b/test/unit/connectors/EC2.py index 91640fa64..bad3c4fa4 100755 --- a/test/unit/connectors/EC2.py +++ b/test/unit/connectors/EC2.py @@ -126,6 +126,7 @@ def test_20_launch(self, save_data, blockdevicemapping, VPCConnection, get_regio cpu.arch='x86_64' and cpu.count>=1 and memory.size>=512m and + instance_tags = 'key=value,key1=value2' and net_interface.0.connection = 'net1' and net_interface.0.dns_name = 'test' and net_interface.1.connection = 'net2' and diff --git a/test/unit/connectors/GCE.py b/test/unit/connectors/GCE.py index d57f4e8cf..2fa3f7d7e 100755 --- a/test/unit/connectors/GCE.py +++ b/test/unit/connectors/GCE.py @@ -133,6 +133,7 @@ def test_20_launch(self, save_data, get_driver): cpu.arch='x86_64' and cpu.count=1 and memory.size=512m and + instance_tags='key=value,key1=value2' and net_interface.0.connection = 'net1' and net_interface.0.dns_name = 'test' and net_interface.1.connection = 'net2' and diff --git a/test/unit/connectors/OpenNebula.py b/test/unit/connectors/OpenNebula.py index 9c60c8e1d..9d94bc2e9 100755 --- a/test/unit/connectors/OpenNebula.py +++ b/test/unit/connectors/OpenNebula.py @@ -134,6 +134,7 @@ def test_20_launch(self, save_data, getONEVersion, server_proxy): net_interface.0.connection = 'net1' and net_interface.0.dns_name = 'test' and net_interface.1.connection = 'net2' and + instance_tags = 'key=value,key1=value2' and disk.0.os.name = 'linux' and disk.0.image.url = 'one://server.com/1' and disk.0.os.credentials.username = 'user' and diff --git a/test/unit/connectors/OpenStack.py b/test/unit/connectors/OpenStack.py index 21c92c77e..4275ea561 100755 --- a/test/unit/connectors/OpenStack.py +++ b/test/unit/connectors/OpenStack.py @@ -145,6 +145,7 @@ def test_20_launch(self, save_data, get_driver): cpu.arch='x86_64' and cpu.count=1 and memory.size=512m and + instance_tags='key=value,key1=value2' and net_interface.0.connection = 'net1' and net_interface.0.dns_name = 'test' and net_interface.1.connection = 'net2' and